{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\u001b[33mWARNING: You are using pip version 20.2.2; however, version 20.2.3 is available.\r\n",
      "You should consider upgrading via the '/opt/conda/bin/python3.7 -m pip install --upgrade pip' command.\u001b[0m\r\n"
     ]
    }
   ],
   "source": [
    "!pip install -q transformers"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "_cell_guid": "b1076dfc-b9ad-4769-8c92-a6c4dae69d19",
    "_uuid": "8f2839f25d086af736a60e9eeb907d3b93b6e0e5"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m W&B installed but not logged in.  Run `wandb login` or set the WANDB_API_KEY env variable.\n"
     ]
    }
   ],
   "source": [
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "from pathlib import Path \n",
    "\n",
    "import os\n",
    "\n",
    "import torch\n",
    "import torch.optim as optim\n",
    "\n",
    "import random \n",
    "\n",
    "# fastai\n",
    "from fastai import *\n",
    "from fastai.text import *\n",
    "from fastai.callbacks import *\n",
    "\n",
    "# transformers\n",
    "from transformers import PreTrainedModel, PreTrainedTokenizer, PretrainedConfig\n",
    "\n",
    "from transformers import BertForSequenceClassification, BertTokenizer, BertConfig\n",
    "from transformers import RobertaForSequenceClassification, RobertaTokenizer, RobertaConfig\n",
    "from transformers import XLNetForSequenceClassification, XLNetTokenizer, XLNetConfig\n",
    "from transformers import XLMForSequenceClassification, XLMTokenizer, XLMConfig\n",
    "from transformers import DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertConfig"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fastai version : 1.0.61\n",
      "transformers version : 2.11.0\n"
     ]
    }
   ],
   "source": [
    "import fastai\n",
    "import transformers\n",
    "print('fastai version :', fastai.__version__)\n",
    "print('transformers version :', transformers.__version__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/kaggle/input/mh-20-fakenews/Participants_Data_WH20/Test.csv\n",
      "/kaggle/input/mh-20-fakenews/Participants_Data_WH20/Train.csv\n",
      "/kaggle/input/mh-20-fakenews/Participants_Data_WH20/sample submission.csv\n"
     ]
    }
   ],
   "source": [
    "for dirname, _, filenames in os.walk('/kaggle/input'):\n",
    "    for filename in filenames:\n",
    "        print(os.path.join(dirname, filename))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "_cell_guid": "79c7e3d0-c299-4dcb-8224-4455121ee9b0",
    "_uuid": "d629ff2d2480ee46fbb7e2d37f6b5fab8052498a"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(10240, 3) (1267, 2)\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Labels</th>\n",
       "      <th>Text</th>\n",
       "      <th>Text_Tag</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Says the Annies List political group supports ...</td>\n",
       "      <td>abortion</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>When did the decline of coal start? It started...</td>\n",
       "      <td>energy,history,job-accomplishments</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Hillary Clinton agrees with John McCain \"by vo...</td>\n",
       "      <td>foreign-policy</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>Health care reform legislation is likely to ma...</td>\n",
       "      <td>health-care</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2</td>\n",
       "      <td>The economic turnaround started at the end of ...</td>\n",
       "      <td>economy,jobs</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Labels                                               Text  \\\n",
       "0       1  Says the Annies List political group supports ...   \n",
       "1       2  When did the decline of coal start? It started...   \n",
       "2       3  Hillary Clinton agrees with John McCain \"by vo...   \n",
       "3       1  Health care reform legislation is likely to ma...   \n",
       "4       2  The economic turnaround started at the end of ...   \n",
       "\n",
       "                             Text_Tag  \n",
       "0                            abortion  \n",
       "1  energy,history,job-accomplishments  \n",
       "2                      foreign-policy  \n",
       "3                         health-care  \n",
       "4                        economy,jobs  "
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train = pd.read_csv('../input/mh-20-fakenews/Participants_Data_WH20/Train.csv')\n",
    "test = pd.read_csv('../input/mh-20-fakenews/Participants_Data_WH20/Test.csv')\n",
    "print(train.shape,test.shape)\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# remove nan in train\n",
    "train = train.drop(index=2142)\n",
    "train = train.drop(index=9375)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "MODEL_CLASSES = {\n",
    "    'bert': (BertForSequenceClassification, BertTokenizer, BertConfig),\n",
    "    'xlnet': (XLNetForSequenceClassification, XLNetTokenizer, XLNetConfig),\n",
    "    'xlm': (XLMForSequenceClassification, XLMTokenizer, XLMConfig),\n",
    "    'roberta': (RobertaForSequenceClassification, RobertaTokenizer, RobertaConfig),\n",
    "    'distilbert': (DistilBertForSequenceClassification, DistilBertTokenizer, DistilBertConfig)\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Parameters\n",
    "seed = 666#42\n",
    "use_fp16 = False\n",
    "bs = 16\n",
    "\n",
    "model_type = 'roberta'\n",
    "pretrained_model_name = 'roberta-base'\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "model_class, tokenizer_class, config_class = MODEL_CLASSES[model_type]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def seed_all(seed_value):\n",
    "    random.seed(seed_value) # Python\n",
    "    np.random.seed(seed_value) # cpu vars\n",
    "    torch.manual_seed(seed_value) # cpu  vars\n",
    "    \n",
    "    if torch.cuda.is_available(): \n",
    "        torch.cuda.manual_seed(seed_value)\n",
    "        torch.cuda.manual_seed_all(seed_value) # gpu vars\n",
    "        torch.backends.cudnn.deterministic = True  #needed\n",
    "        torch.backends.cudnn.benchmark = False"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed_all(seed)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class TransformersBaseTokenizer(BaseTokenizer):\n",
    "    \"\"\"Wrapper around PreTrainedTokenizer to be compatible with fast.ai\"\"\"\n",
    "    def __init__(self, pretrained_tokenizer: PreTrainedTokenizer, model_type = 'bert', **kwargs):\n",
    "        self._pretrained_tokenizer = pretrained_tokenizer\n",
    "        self.max_seq_len = pretrained_tokenizer.max_len\n",
    "        self.model_type = model_type\n",
    "\n",
    "    def __call__(self, *args, **kwargs): \n",
    "        return self\n",
    "\n",
    "    def tokenizer(self, t:str) -> List[str]:\n",
    "        \"\"\"Limits the maximum sequence length and add the spesial tokens\"\"\"\n",
    "        CLS = self._pretrained_tokenizer.cls_token\n",
    "        SEP = self._pretrained_tokenizer.sep_token\n",
    "        if self.model_type in ['roberta']:\n",
    "            tokens = self._pretrained_tokenizer.tokenize(t, add_prefix_space=True)[:self.max_seq_len - 2]\n",
    "            tokens = [CLS] + tokens + [SEP]\n",
    "        else:\n",
    "            tokens = self._pretrained_tokenizer.tokenize(t)[:self.max_seq_len - 2]\n",
    "            if self.model_type in ['xlnet']:\n",
    "                tokens = tokens + [SEP] +  [CLS]\n",
    "            else:\n",
    "                tokens = [CLS] + tokens + [SEP]\n",
    "        return tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "6071e41711854f4f82636631df0e7d87",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=898823.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a77e47b86add4394b699a9cc0de4854f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=456318.0, style=ProgressStyle(descripti…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "transformer_tokenizer = tokenizer_class.from_pretrained(pretrained_model_name)\n",
    "transformer_base_tokenizer = TransformersBaseTokenizer(pretrained_tokenizer = transformer_tokenizer, model_type = model_type)\n",
    "fastai_tokenizer = Tokenizer(tok_func = transformer_base_tokenizer, pre_rules=[], post_rules=[])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "class TransformersVocab(Vocab):\n",
    "    def __init__(self, tokenizer: PreTrainedTokenizer):\n",
    "        super(TransformersVocab, self).__init__(itos = [])\n",
    "        self.tokenizer = tokenizer\n",
    "    \n",
    "    def numericalize(self, t:Collection[str]) -> List[int]:\n",
    "        \"Convert a list of tokens `t` to their ids.\"\n",
    "        return self.tokenizer.convert_tokens_to_ids(t)\n",
    "        #return self.tokenizer.encode(t)\n",
    "\n",
    "    def textify(self, nums:Collection[int], sep=' ') -> List[str]:\n",
    "        \"Convert a list of `nums` to their tokens.\"\n",
    "        nums = np.array(nums).tolist()\n",
    "        return sep.join(self.tokenizer.convert_ids_to_tokens(nums)) if sep is not None else self.tokenizer.convert_ids_to_tokens(nums)\n",
    "    \n",
    "    def __getstate__(self):\n",
    "        return {'itos':self.itos, 'tokenizer':self.tokenizer}\n",
    "\n",
    "    def __setstate__(self, state:dict):\n",
    "        self.itos = state['itos']\n",
    "        self.tokenizer = state['tokenizer']\n",
    "        self.stoi = collections.defaultdict(int,{v:k for k,v in enumerate(self.itos)})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "transformer_vocab =  TransformersVocab(tokenizer = transformer_tokenizer)\n",
    "numericalize_processor = NumericalizeProcessor(vocab=transformer_vocab)\n",
    "\n",
    "tokenize_processor = TokenizeProcessor(tokenizer=fastai_tokenizer, include_bos=False, include_eos=False)\n",
    "\n",
    "transformer_processor = [tokenize_processor, numericalize_processor]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "pad_first = bool(model_type in ['xlnet'])\n",
    "pad_idx = transformer_tokenizer.pad_token_id"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Sal', 'ut', 'Ġc', 'Ġest', 'Ġmo', 'i', ',', 'ĠHello', 'Ġit', 'Ġs', 'Ġme']\n",
      "[18111, 1182, 740, 3304, 7458, 118, 6, 20920, 24, 579, 162]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "['Sal', 'ut', 'Ġc', 'Ġest', 'Ġmo', 'i', ',', 'ĠHello', 'Ġit', 'Ġs', 'Ġme']"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tokens = transformer_tokenizer.tokenize('Salut c est moi, Hello it s me')\n",
    "print(tokens)\n",
    "ids = transformer_tokenizer.convert_tokens_to_ids(tokens)\n",
    "print(ids)\n",
    "transformer_tokenizer.convert_ids_to_tokens(ids)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train['combine'] = train['Text']+' about '+ train['Text_Tag']\n",
    "test['combine'] = test['Text']+' about '+ test['Text_Tag']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "databunch = (TextList.from_df(train, cols='combine', processor=transformer_processor)\n",
    "             .split_by_rand_pct(0.1,seed=seed)\n",
    "             .label_from_df(cols= 'Labels')\n",
    "             .add_test(test)\n",
    "             .databunch(bs=bs, pad_first=pad_first, pad_idx=pad_idx))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS] token : <s>\n",
      "[SEP] token : </s>\n",
      "[PAD] token : <pad>\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>text</th>\n",
       "      <th>target</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>&lt;s&gt; ĠThe Ġvast Ġmajority Ġof Ġthe Ġmoney ĠI Ġgot Ġwas Ġfrom Ġsmall Ġdonors Ġall Ġacross Ġthe Ġcountry .'' ĉ campaign - f inance ĉ bar ack - ob ama ĉ President ĉ Ill inois ĉ dem ocrat ĉ 70 ĉ 71 ĉ 160 ĉ 163 ĉ 9 ĉ an Ġinterview Ġwith ĠCNBC 's ĠJohn ĠHar wood Ġ Ċ 5 802 . json ĉ true ĉ The ĠDemocrat - controlled ĠSenate</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>&lt;s&gt; ĠPeople Ġwant Ġthe Ġminimum Ġwage , Ġthey Ġwant Ġmarriage Ġequality , Ġthey Ġwant Ġwom ens Ġhealth Ġcare . ĠThe res Ġissues Ġthat Ġactually Ġthat Ġreally Ġdo Ġmatter , Ġsocial Ġissues Ġthat Ġmatter Ġto Ġthe Ġpeople Ġof Ġthis Ġstate . ĠSo ĠI Ġknow ĠBarbara Ġwould Ġsupport Ġthose Ġissues , ĠI Ġknow Ġthe Ġgovernor Ġhas nt . ĠYou ve Ġseen Ġthe Ġactions Ġhes Ġtaken Ġon Ġthem . Ġabout Ġgays - and</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>&lt;s&gt; Ġ[ When ĠI Ġwas Ġmayor ] ĠProvidence Ġwas Ġone Ġof Ġthe Ġfive Ġrenaissance Ġcities Ġof ĠAmerica , Ġaccording Ġto ĠUSA ĠToday Ġin Ġthose Ġdays . ĠMoney Ġmagazine Ġsaid Ġit Ġwas Ġthe Ġfifth Ġbest Ġcity Ġto Ġlive Ġin Ġin ĠAmerica . ĠIn Ġaddition Ġto Ġthat , ĠAll ĠCities ĠAl manac Ġsaid Ġin Ġ1994 , ĠI Ġbelieve , Ġthat Ġit Ġwas Ġthe Ġsafest Ġcity Ġin ĠAmerica . Ġabout Ġcandidates - bi</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>&lt;s&gt; ĠThe ĠW IC Ġprogram Ġis Ġan Ġincredibly Ġcost Ġeffective Ġprogram , Ġserving Ġnearly Ġ10 Ġmillion ĠAmericans Ġeach Ġyear Ġand Ġcosting Ġless Ġthan Ġ$ 100 Ġper Ġperson . ĠIn Ġmy Ġdistrict , Ġmore Ġthan Ġ18 Ġpercent Ġof Ġresidents Ġsuffer Ġfrom Ġfood Ġinsecurity Ġand Ġdepend Ġon ĠW IC Ġto Ġmake Ġends Ġmeet . Ġabout Ġchildren , f ederal - budget , health - care , hun ger , p overty ,</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>&lt;s&gt; ĠSays Ġits Ġa Ġstatistical Ġreality Ġthat Ġmost Ġsingle Ġmoms Ġare Ġvery Ġpoor , Ġunder - educated , Ġcant Ġget Ġa Ġjob , Ġand Ġif Ġit Ġwere nt Ġfor Ġgovernment Ġassistance , Ġtheir Ġkids Ġwould Ġbe Ġstarving Ġto Ġdeath . Ġabout Ġchildren , econom y , education , fam ilies , health - care , hun ger , income , jobs , l abor , marriage , p overty ,</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "print('[CLS] token :', transformer_tokenizer.cls_token)\n",
    "print('[SEP] token :', transformer_tokenizer.sep_token)\n",
    "print('[PAD] token :', transformer_tokenizer.pad_token)\n",
    "databunch.show_batch()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[CLS] id : 0\n",
      "[SEP] id : 2\n",
      "[PAD] id : 1\n",
      "Batch shape :  torch.Size([16, 512])\n",
      "tensor([[    0,    20,  4714,  ..., 50117,  4809,     2],\n",
      "        [    0,    96, 12374,  ...,     1,     1,     1],\n",
      "        [    0,    20,   305,  ...,     1,     1,     1],\n",
      "        ...,\n",
      "        [    0,  1590,  6542,  ...,     1,     1,     1],\n",
      "        [    0, 15674, 16272,  ...,     1,     1,     1],\n",
      "        [    0, 15674,   114,  ...,     1,     1,     1]])\n"
     ]
    }
   ],
   "source": [
    "print('[CLS] id :', transformer_tokenizer.cls_token_id)\n",
    "print('[SEP] id :', transformer_tokenizer.sep_token_id)\n",
    "print('[PAD] id :', pad_idx)\n",
    "test_one_batch = databunch.one_batch()[0]\n",
    "print('Batch shape : ',test_one_batch.shape)\n",
    "print(test_one_batch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# defining our model architecture \n",
    "class CustomTransformerModel(nn.Module):\n",
    "    def __init__(self, transformer_model: PreTrainedModel):\n",
    "        super(CustomTransformerModel,self).__init__()\n",
    "        self.transformer = transformer_model\n",
    "        \n",
    "    def forward(self, input_ids, attention_mask=None):\n",
    "        \n",
    "        # attention_mask\n",
    "        # Mask to avoid performing attention on padding token indices.\n",
    "        # Mask values selected in ``[0, 1]``:\n",
    "        # ``1`` for tokens that are NOT MASKED, ``0`` for MASKED tokens.\n",
    "        attention_mask = (input_ids!=pad_idx).type(input_ids.type()) \n",
    "        \n",
    "        logits = self.transformer(input_ids,\n",
    "                                  attention_mask = attention_mask)[0]   \n",
    "        return logits"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "4be73dd3eca74a159a2566185136d145",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=481.0, style=ProgressStyle(description_…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "RobertaConfig {\n",
      "  \"architectures\": [\n",
      "    \"RobertaForMaskedLM\"\n",
      "  ],\n",
      "  \"attention_probs_dropout_prob\": 0.1,\n",
      "  \"bos_token_id\": 0,\n",
      "  \"eos_token_id\": 2,\n",
      "  \"hidden_act\": \"gelu\",\n",
      "  \"hidden_dropout_prob\": 0.1,\n",
      "  \"hidden_size\": 768,\n",
      "  \"id2label\": {\n",
      "    \"0\": \"LABEL_0\",\n",
      "    \"1\": \"LABEL_1\",\n",
      "    \"2\": \"LABEL_2\",\n",
      "    \"3\": \"LABEL_3\",\n",
      "    \"4\": \"LABEL_4\",\n",
      "    \"5\": \"LABEL_5\"\n",
      "  },\n",
      "  \"initializer_range\": 0.02,\n",
      "  \"intermediate_size\": 3072,\n",
      "  \"label2id\": {\n",
      "    \"LABEL_0\": 0,\n",
      "    \"LABEL_1\": 1,\n",
      "    \"LABEL_2\": 2,\n",
      "    \"LABEL_3\": 3,\n",
      "    \"LABEL_4\": 4,\n",
      "    \"LABEL_5\": 5\n",
      "  },\n",
      "  \"layer_norm_eps\": 1e-05,\n",
      "  \"max_position_embeddings\": 514,\n",
      "  \"model_type\": \"roberta\",\n",
      "  \"num_attention_heads\": 12,\n",
      "  \"num_hidden_layers\": 12,\n",
      "  \"pad_token_id\": 1,\n",
      "  \"type_vocab_size\": 1,\n",
      "  \"vocab_size\": 50265\n",
      "}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "config = config_class.from_pretrained(pretrained_model_name)\n",
    "config.num_labels = 6\n",
    "config.use_bfloat16 = use_fp16\n",
    "print(config)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "78e61d102def4e3f852738f1ad7d19b8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, description='Downloading', max=501200538.0, style=ProgressStyle(descri…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "transformer_model = model_class.from_pretrained(pretrained_model_name, config = config)\n",
    "# transformer_model = model_class.from_pretrained(pretrained_model_name, num_labels = 6)\n",
    "\n",
    "custom_transformer_model = CustomTransformerModel(transformer_model = transformer_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "from fastai.callbacks import *\n",
    "from transformers import AdamW\n",
    "from functools import partial\n",
    "\n",
    "CustomAdamW = partial(AdamW, correct_bias=False)\n",
    "\n",
    "learner = Learner(databunch, \n",
    "                  custom_transformer_model, \n",
    "                  opt_func = CustomAdamW, \n",
    "                  metrics=[accuracy, error_rate])\n",
    "\n",
    "# Show graph of learner stats and metrics after each epoch.\n",
    "learner.callbacks.append(ShowGraph(learner))\n",
    "\n",
    "# Put learn in FP16 precision mode. --> Seems to not working\n",
    "if use_fp16: learner = learner.to_fp16()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "CustomTransformerModel(\n",
      "  (transformer): RobertaForSequenceClassification(\n",
      "    (roberta): RobertaModel(\n",
      "      (embeddings): RobertaEmbeddings(\n",
      "        (word_embeddings): Embedding(50265, 768, padding_idx=1)\n",
      "        (position_embeddings): Embedding(514, 768, padding_idx=1)\n",
      "        (token_type_embeddings): Embedding(1, 768)\n",
      "        (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "        (dropout): Dropout(p=0.1, inplace=False)\n",
      "      )\n",
      "      (encoder): BertEncoder(\n",
      "        (layer): ModuleList(\n",
      "          (0): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (1): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (2): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (3): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (4): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (5): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (6): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (7): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (8): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (9): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (10): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "          (11): BertLayer(\n",
      "            (attention): BertAttention(\n",
      "              (self): BertSelfAttention(\n",
      "                (query): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (key): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (value): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "              (output): BertSelfOutput(\n",
      "                (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "                (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "                (dropout): Dropout(p=0.1, inplace=False)\n",
      "              )\n",
      "            )\n",
      "            (intermediate): BertIntermediate(\n",
      "              (dense): Linear(in_features=768, out_features=3072, bias=True)\n",
      "            )\n",
      "            (output): BertOutput(\n",
      "              (dense): Linear(in_features=3072, out_features=768, bias=True)\n",
      "              (LayerNorm): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "              (dropout): Dropout(p=0.1, inplace=False)\n",
      "            )\n",
      "          )\n",
      "        )\n",
      "      )\n",
      "      (pooler): BertPooler(\n",
      "        (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "        (activation): Tanh()\n",
      "      )\n",
      "    )\n",
      "    (classifier): RobertaClassificationHead(\n",
      "      (dense): Linear(in_features=768, out_features=768, bias=True)\n",
      "      (dropout): Dropout(p=0.1, inplace=False)\n",
      "      (out_proj): Linear(in_features=768, out_features=6, bias=True)\n",
      "    )\n",
      "  )\n",
      ")\n"
     ]
    }
   ],
   "source": [
    "print(learner.model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# For roberta-base\n",
    "list_layers = [learner.model.transformer.roberta.embeddings,\n",
    "              learner.model.transformer.roberta.encoder.layer[0],\n",
    "              learner.model.transformer.roberta.encoder.layer[1],\n",
    "              learner.model.transformer.roberta.encoder.layer[2],\n",
    "              learner.model.transformer.roberta.encoder.layer[3],\n",
    "              learner.model.transformer.roberta.encoder.layer[4],\n",
    "              learner.model.transformer.roberta.encoder.layer[5],\n",
    "              learner.model.transformer.roberta.encoder.layer[6],\n",
    "              learner.model.transformer.roberta.encoder.layer[7],\n",
    "              learner.model.transformer.roberta.encoder.layer[8],\n",
    "              learner.model.transformer.roberta.encoder.layer[9],\n",
    "              learner.model.transformer.roberta.encoder.layer[10],\n",
    "              learner.model.transformer.roberta.encoder.layer[11],\n",
    "              learner.model.transformer.roberta.pooler]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Learner split in 14 groups\n",
      "[Sequential(\n",
      "  (0): Embedding(50265, 768, padding_idx=1)\n",
      "  (1): Embedding(514, 768, padding_idx=1)\n",
      "  (2): Embedding(1, 768)\n",
      "  (3): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (4): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (5): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (6): Dropout(p=0.1, inplace=False)\n",
      "  (7): Linear(in_features=768, out_features=3072, bias=True)\n",
      "  (8): Linear(in_features=3072, out_features=768, bias=True)\n",
      "  (9): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
      "  (10): Dropout(p=0.1, inplace=False)\n",
      "), Sequential(\n",
      "  (0): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (1): Tanh()\n",
      "  (2): Linear(in_features=768, out_features=768, bias=True)\n",
      "  (3): Dropout(p=0.1, inplace=False)\n",
      "  (4): Linear(in_features=768, out_features=6, bias=True)\n",
      ")]\n"
     ]
    }
   ],
   "source": [
    "learner.split(list_layers)\n",
    "num_groups = len(learner.layer_groups)\n",
    "print('Learner split in',num_groups,'groups')\n",
    "print(learner.layer_groups)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.save('untrain')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed_all(seed)\n",
    "learner.load('untrain');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.freeze_to(-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CustomTransformerModel\n",
       "======================================================================\n",
       "Layer (type)         Output Shape         Param #    Trainable \n",
       "======================================================================\n",
       "Embedding            [512, 768]           38,603,520 False     \n",
       "______________________________________________________________________\n",
       "Embedding            [512, 768]           394,752    False     \n",
       "______________________________________________________________________\n",
       "Embedding            [512, 768]           768        False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "Dropout              [12, 512, 512]       0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           590,592    False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 3072]          2,362,368  False     \n",
       "______________________________________________________________________\n",
       "Linear               [512, 768]           2,360,064  False     \n",
       "______________________________________________________________________\n",
       "LayerNorm            [512, 768]           1,536      False     \n",
       "______________________________________________________________________\n",
       "Dropout              [512, 768]           0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [768]                590,592    True      \n",
       "______________________________________________________________________\n",
       "Tanh                 [768]                0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [768]                590,592    True      \n",
       "______________________________________________________________________\n",
       "Dropout              [768]                0          False     \n",
       "______________________________________________________________________\n",
       "Linear               [6]                  4,614      True      \n",
       "______________________________________________________________________\n",
       "\n",
       "Total params: 125,240,838\n",
       "Total trainable params: 1,185,798\n",
       "Total non-trainable params: 124,055,040\n",
       "Optimized with 'transformers.optimization.AdamW', correct_bias=False\n",
       "Using true weight decay as discussed in https://www.fast.ai/2018/07/02/adam-weight-decay/ \n",
       "Loss function : FlattenedLoss\n",
       "======================================================================\n",
       "Callbacks functions applied \n",
       "    ShowGraph"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "learner.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='68' class='' max='575' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      11.83% [68/575 00:02<00:21 3.0598]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n"
     ]
    }
   ],
   "source": [
    "learner.lr_find()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Min numerical gradient: 1.10E-06\n",
      "Min loss divided by 10: 3.98E-07\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZAAAAEGCAYAAABLgMOSAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deXxV1b338c8vExAygQljmGcEAY2KioozDtXWtrZWbaVWL9Vaq71ebW21re1tn/tcbe2jraWt4lBRa2nrgCPVoqIgaCCQoEIYEoYMQCYy56znj3PQgBlPsrPPSb7v1ysvcvZwzu8sTvLN2nuvtc05h4iISGfF+F2AiIhEJwWIiIiERQEiIiJhUYCIiEhYFCAiIhKWOL8L6E7p6elu7NixfpchIhI11q1bV+qcywhn314VIGPHjmXt2rV+lyEiEjXMbEe4++oQloiIhEUBIiIiYVGAiIhIWBQgIiISFgWIiIiERQEiIiJh8SxAzOwhMys2s42trB9kZn83sw1mtsbMZoSWjzKz180sz8w2mdlNXtUoIiLh87IHsgRY0Mb6HwLZzrljgK8D94WWNwLfd85NA+YCN5jZdA/rFBGJWm9+XMKSt7fR0BTo8df2LECccyuB/W1sMh1YEdp2MzDWzIY65/Y4594PLa8E8oCRXtUpIhKtmgKOu5/P5ZF3wh4L2CV+ngNZD1wKYGYnAGOAzOYbmNlYYA6wurUnMbPrzGytma0tKSnxrFgRkUiz7P1CPiqq4tbzphAf2/O/zv0MkF8Bg8wsG7gR+IDg4SsAzCwJ+BvwPedcRWtP4pxb7JzLcs5lZWSENZ2LiEjUqW1o4t5XP2LWqDTOnzHMlxp8mwsrFAoLAczMgG2hL8wsnmB4/MU5t8yvGkVEItUjq7azp7yWX39lNsFfoT3Ptx6ImaWZWULo4beAlc65ilCY/BnIc87d61d9IiKRqqy6ngde38IZUzKYO/4o3+rwrAdiZkuB+UC6mRUCdwHxAM65B4FpwKNm1gTkAteEdj0FuArICR3eAvihc265V7WKiEST37+xlcq6Rm47f6qvdXgWIM65y9tZ/w4wqYXlbwH+9MdERCLc7rIaHl61nUvnZDJ1WIqvtWgkuohIFLn31Y8AuOXcyT5XogAREYkam/dW8Lf3C7n65LGMTBvgdzkKEBGRaPE/L31IUr84rp8/we9SAAWIiEhUeDd/H//aXMz18yeSlpjQ/g49QAEiIhLhiitqufmpbEamDWDhKWP9LucTvg0kFBGR9tXUN/GtR9dSXtPAXxedRP/4WL9L+oQCREQkQgUCjlueziZnVzl/vCqLo0ek+l3SYXQIS0QkQv3fVz7kxY17ueOCaZw9fajf5XyGAkREJAI9vbaA37+xlctPGM0188b5XU6LFCAiIhHm3fx93PH3HOZNTOdnlxzt22SJ7VGAiIhEkO2lB1n0+DpGD07kgSuO9eU+Hx2lk+giIhHkiTU7qa5r4p83HE/qgHi/y2lT5EabiEgftKe8lhFp/Rlz1EC/S2mXAkREJIIUVdQyJLm/32V0iAJERCSClFTWMSSln99ldIgCREQkghRV1DI0RT0QERHphKq6RqrrmxiSrB6IiIh0QlFFLYB6ICIi0jmHAkTnQEREpFNKKusAdBWWiIh0zqeHsPp4D8TMHjKzYjPb2Mr6QWb2dzPbYGZrzGxGs3ULzOxDM9tiZrd7VaOISCQpqqgjMSGWpH7RMUmIlz2QJcCCNtb/EMh2zh0DfB24D8DMYoEHgPOB6cDlZjbdwzpFRCJCcWUdQ5L7RezkiUfyLECccyuB/W1sMh1YEdp2MzDWzIYCJwBbnHP5zrl64EngEq/qFBGJFEUVtQyJkiuwwN9zIOuBSwHM7ARgDJAJjAQKmm1XGFrWIjO7zszWmtnakpISD8sVEfFWcUVt1IwBAX8D5FfAIDPLBm4EPgAagZb6bq61J3HOLXbOZTnnsjIyMrypVETEY845iivromYMCPg4nbtzrgJYCGDBA37bQl+JwKhmm2YCu3u8QBGRHnRoFHq0XIEFPvZAzCzNzBJCD78FrAyFynvAJDMbF1r/VeBZv+oUEekJRRXRNQYEPOyBmNlSYD6QbmaFwF1APIBz7kFgGvComTUBucA1oXWNZvYd4GUgFnjIObfJqzpFRCJBcWV0jUIHDwPEOXd5O+vfASa1sm45sNyLukREIlFxqAcSTedANBJdRCQCfDIPlq7CEhGRziiujK5R6KAAERGJCIduJBUto9BBASIiEhGKK+rIiKLDV6AAERGJCMWV0XMr20MUICIiPnPOUVRRx1D1QEREpDMq6xqpaWiKqjEgoAAREfFdNI4BAQWIiIjvij8ZA6IAERGRTiiKwmlMQAEiIuI7HcISEZGwRNu90A9RgIiI+KwoCseAgAJERMR3JRV1UTWJ4iEKEBERnxVV1jJEPRAREemM4Cj02qgbhQ4KEBERX1XWNVLbENA5EBER6ZxPBhFG2RgQUICIiPiqKDQGJNpGoYMCRETEV8WhUehD1QMREZHO+KQHonMgnzKzh8ys2Mw2trI+1cyeM7P1ZrbJzBY2W3dzaNlGM1tqZtHXsiIiHVBUUcvAKByFDt72QJYAC9pYfwOQ65ybBcwH7jGzBDMbCXwXyHLOzQBiga96WKeIiG+KK+ui8gos8DBAnHMrgf1tbQIkW/AO8kmhbRtD6+KAAWYWByQCu72qU0TET8UVtVF3L/RD/DwHcj8wjWA45AA3OecCzrldwP8CO4E9QLlz7pXWnsTMrjOztWa2tqSkpCfqFhHpNuqBhOc8IBsYAcwG7jezFDMbBFwCjAutG2hmV7b2JM65xc65LOdcVkZGRk/ULSLSLT4ZhR6FV2CBvwGyEFjmgrYA24CpwNnANudciXOuAVgGnOxjnSIinqioDY5Cj8YxIOBvgOwEzgIws6HAFCA/tHyumSWGzo+cBeT5VqWIiEdKovROhId4dt2YmS0leHVVupkVAncB8QDOuQeBu4ElZpYDGHCbc64UKDWzZ4D3CZ5U/wBY7FWdIiJ+KYrSOxEe4lmAOOcub2f9buDcVtbdRTBwRER6raJD82DpKiwREemM4sroHYUOChAREd9E8yh0UICIiPimuCJ6x4CAAkRExDfFlbVRewUWKEBERHxTVFEXtWNAQAEiIuKLaB+FDgoQERFP1TU28eHeys8sr6htpK4xOu+FfogCRETEQw+9tZ3zfrOSqx9ew5biqk+WH7oXerTOxAsKEBERT723fT9pifGs236ABb9Zyc+ey6W8piHqR6GDhyPRRUT6OuccGwrLOGvqUH5wwVTueeUjHl61jX9k7+KEsYOB6A4Q9UBERDyyu7yW0qp6Zo1KJT2pH7+8dCbP3ziPSUOSeGnTXiB6pzEB9UBERDyzoaAMgGMy0z5ZdvSIVJ68bi4vb9rLrrJaBkbpKHRQgIiIeCa7sIz4WGPa8OTDlpsZC2YM96mq7qNDWCIiHtlQUM604Sn0i4v1uxRPKEBERDwQCDhydpVzTGaq36V4RgEiIuKB/NIqquoamdXs/EdvowAREfHA+oJyAGaNUoCIiEgnbCgsIzEhlgkZSX6X4hkFiIiIB7ILy5k5MpXYGPO7FM8oQEREull9Y4C83RW9+vAVKEBERLrdh3srqW8K9OorsMDDADGzh8ys2Mw2trI+1cyeM7P1ZrbJzBY2W5dmZs+Y2WYzyzOzk7yqU0Sku2UXBkeg9+YrsMDbHsgSYEEb628Acp1zs4D5wD1mlhBadx/wknNuKjALyPOwThGRbrWhoIzBAxPIHDTA71I85VmAOOdWAvvb2gRINjMDkkLbNppZCnAa8OfQ89Q758q8qlNEpLttKAwOIAz+euu9OhQgZjbQzGJC3082s4vNLL6Lr30/MA3YDeQANznnAsB4oAR42Mw+MLM/mdnANmq7zszWmtnakpKSLpYkItI1B+sa+bi48rAJFHurjvZAVgL9zWwksAJYSPAQVVecB2QDI4DZwP2h3kcccCzwe+fcHOAgcHtrT+KcW+ycy3LOZWVkZHSxJBGRrtm4q5yAg9mjevcJdOh4gJhzrhq4FPh/zrkvANO7+NoLgWUuaAuwDZgKFAKFzrnVoe2eIRgoIiIRb0NhcAS6eiCfstCVUFcAL4SWdXUq+J3AWaEnHwpMAfKdc3uBAjObEtruLCC3i68lItIj1heWMTJtAOlJ0XujqI7qaAh8D/gB8Hfn3CYzGw+83tYOZraU4NVV6WZWCNwFxAM45x4E7gaWmFkOYMBtzrnS0O43An8JXZWVT7C3IiIS8dYXljGrDxy+gg4GiHPu38C/AUIn00udc99tZ5/L21m/Gzi3lXXZQFZHahMRiRT7D9ZTsL+GK04c43cpPaKjV2E9YWYpoauhcoEPzexWb0sTEYkuGwoP3cK2b/RAOnoOZLpzrgL4PLAcGA1c5VlVIiJRaH1BOWYwc6QCpLn40LiPzwP/dM41EBwIKCIiIRsKy5iQkURy/64Ok4sOHQ2QPwDbgYHASjMbA1R4VZSISLRxzrG+sHffwvZIHT2J/lvgt80W7TCzM7wpSUQk+uwpr6W0qo7ZvXwK9+Y6ehI91czuPTRliJndQ7A3IiIiwPqCQyfQFSBHegioBC4LfVUAD3tVlIhItHk1r4jkfnFMG57sdyk9pqMDCSc4577Y7PFPzSzbi4JERKJNVV0jL+bs5fNzRtAvLtbvcnpMR3sgNWY279ADMzsFqPGmJBGR6LI8Zw81DU186bhMv0vpUR3tgSwCHjWzQ5cXHAC+4U1JIiLR5Zl1hYxPH8ixowf5XUqP6lAPxDm3PnTnwGOAY0LTrJ/paWUiIlFgx76DrNm2ny8el9nrbyB1pE7dkdA5VxEakQ5wiwf1iIhElb+9vwszuPTYkX6X0uO6ckvbvhW1IiJHCAQcf1tXyLyJ6QxP7d33P29JVwJEU5mISJ/27rZ97Cqr6XMnzw9p8yS6mVXSclAY0PfiVkSkmWfWFZLcL47zjh7mdym+aDNAnHN9Z0SMiEgnfDr2YyT94/vO2I/munIIS0Skz+qrYz+aU4CIiITh07EffWfuqyMpQEREOqkvj/1oTgEiItJJfXnsR3MKEBGRTujrYz+a8yxAzOwhMys2s42trE81s+fMbL2ZbTKzhUesjzWzD8zsea9qFBHprLe3lvbpsR/NedkDWQIsaGP9DUBuaI6t+cA9ZpbQbP1NQJ5n1YmIdFJDU4CfP5/H8NT+fXbsR3OeBYhzbiWwv61NgGQLnoFKCm3bCGBmmcCFwJ+8qk9EpLMWr8znw6JK7r5kRp8d+9Gcn+dA7gemAbuBHOAm51wgtO43wH8BgVb2/YSZXXfoVrslJSWeFSsifVt+SRX3rfiYC2cO5+zpQ/0uJyL4GSDnAdnACGA2cL+ZpZjZRUCxc25dR57EObfYOZflnMvKyMjwsFwR6aucc/zw7zn0i4vhrs9N97uciOFngCwElrmgLcA2YCpwCnCxmW0HngTONLPH/StTRPq6v64t5N38/fzwgmkMSenvdzkRw88A2QmcBWBmQ4EpQL5z7gfOuUzn3Fjgq8C/nHNX+lemiPRlJZV1/GJ5HieMG8xXskb5XU5E6egtbTvNzJYSvLoq3cwKgbuAeADn3IPA3cASM8shOLvvbc65Uq/qEREJx8+ez6Wmvon//sJMYmL67qjzlngWIM65y9tZvxs4t51t3gDe6L6qREQ67vXNxTy3fje3nDOZiUOS/C4n4mgkuohICw7WNfKjf2xk0pAkFp0+we9yIpJnPRARkWj22Ls72FVWwzOLTiIhTn9rt0StIiJyhEDA8ZfVOzhx3GCyxg72u5yIpQARETnCyo9LKNhfw5Vzx/hdSkRTgIiIHOHxd3eSnpSg+a7aoQAREWlmV1kN/9pcxGVZo3Tuox1qHRGRZp5csxMHXH7CaL9LiXgKEBGRkIamAE++V8AZU4YwanCi3+VEPAWIiEjIq7lFlFTWccWJ6n10hAJERCTk8Xd3MDJtAPOnDPG7lKigABERAbaWVLFq6z6+duJoYjXnVYcoQEREgCdW7yQuxrhMM+52mAJERPq82oYmnllXyHkzhpGR3M/vcqKGAkRE+rzn1u+mvKaBK0/UyPPOUICISJ/3+OqdTMgYyNzxmveqMxQgItKnvbBhD+sLyrjixDGY6eR5Z2g6dxHpkw4crOenz23iH9m7mTosmS9lZfpdUtRRgIhIn7M8Zw93/nMjZdUN3HTWJG44Y6LmvQqDAkRE+oySyjru/OdGXty4l5kjU3nsmhOZNjzF77KilgJERPqEtz4u5TtL36e6vonbFkzl2lPHERerXkdXKEBEpNf7Z/Yu/vOv65mQkcT9XzuWiUOS/C6pV/Asfs3sITMrNrONraxPNbPnzGy9mW0ys4Wh5aPM7HUzywstv8mrGkWk9/vjynxuejKb48YM4ulFJyk8upGX/bclwII21t8A5DrnZgHzgXvMLAFoBL7vnJsGzAVuMLPpHtYpIr1QIOC4+/lcfrE8jwuPGc4j3zyBlP7xfpfVq3h2CMs5t9LMxra1CZBswQuvk4D9QKNzbg+wJ/QclWaWB4wEcr2qVUR6l7rGJm796waeXb+bq08ey50XTSdGEyR2Oz/PgdwPPAvsBpKBrzjnAs03CAXQHGB1a09iZtcB1wGMHq05/EX6uq0lVfz4HxtZtXUft58/lf84bbwGCHrEzwA5D8gGzgQmAK+a2ZvOuQoAM0sC/gZ879CyljjnFgOLAbKyspznVYtIxKlvDPBK7l7+8u5O3snfR0JsDPdeNotLj9XgQC/5GSALgV855xywxcy2AVOBNWYWTzA8/uKcW+ZjjSISwXbuq+aJNTt5Zl0BpVX1ZA4awK3nTeHLWZkMSe7vd3m9np8BshM4C3jTzIYCU4D80DmRPwN5zrl7faxPRCJY3p4KLr7/LZoCjrOmDeWKE0dz2qQMnevoQZ4FiJktJXh1VbqZFQJ3AfEAzrkHgbuBJWaWAxhwm3Ou1MzmAVcBOWaWHXq6HzrnlntVq4hEn+c37Cbg4I3/PIPRRyX6XU6f5OVVWJe3s343cG4Ly98iGCgiIq1akVdM1phBCg8faRy/iESdwgPVbN5bydnThvpdSp+mABGRqLMirxiAs6YN8bmSvk0BIiJR57W8IsanD2R8hqYl8ZMCRESiSlVdI6vz96v3EQEUICISVd78qIT6pgBn6fyH7xQgIhJVXssrJnVAPFljBvldSp+nABGRqNEUcLz+YTHzp2ToZlARQP8DIhI1sgsOsP9gvQ5fRQgFiIhEjdfyiomLMU6fnOF3KYICRESiyIq8Io4fO5jUAboxVCRQgIiI7+obA/xrcxH/vTyP3WU1LW5TsL+aj4qqdPluBPFzNt6Ita+qjkfe2cHT7xUwdXgyty2YyrThKX6XJRJV6hsDLHu/kLjYGMalD2Rc+kAGJcZ/cnOn+sYAb20p4YUNe3kldy+VtY0AvJZbxNOLTiI9qd9hz/daXhGAzn9EEAVIMzv3VfPHN/N5em0BdY0BTp2Uzvs7DnDBb9/kC3NG8v1zpzAybYDfZYpEvAMH61n0+DpWb9t/2PKU/nGMSx9IRnJ/1mzbR0VtI8n94zjv6GFcOHM4/eNjWbhkDV//8xqe/I+5h93DfEVeMeMzgkEkkUEBAuQUlvOHlVtZnrOH2BjjC3NGct1p45k4JJny6gZ+98YWHl61nec37OEbJ43hhjMmkpaY4HfZIp7ZUlzJiLQBJCZ0/lfEx0WVXPPIWvZW1HLvZbOYPSqN7fsOkl9ykO37DrK9tJr8kirOmT6MC48ZxryJGSTEfXo0/Q9XZfGtR97jmiXv8eg3T2RAQiyVtQ2s3raPhaeM6863KV1kwRsC9g5ZWVlu7dq1ndqnoraBE3+xgrgY42tzR/PNU8YxNOWzdzLbVVbDva98xLIPCknuF8f9XzuW03QliPRCJZV1nPyrFcwcmcoT186lf3xsh/d9/cNivvvEB/SLj2Xx14/j2NHhDfZ7fsNublz6AadPzmDxVVm8mlvEDU+8z1PXzeXE8UeF9ZzSMjNb55zLCmvfvh4gAG99XMoxo1IP6y63ZvPeCm584gPKaxp49ZbTdTWI9DpPrtnJ7ctyADjv6KH87orjiG3nLn/OOR56ezu/eCGXqcNS+NM3shjRxcO9S9fs5AfLcvjcrBHEGrz+YQnrfnS2BhB2s64EiP4ngHmT0jsUHgBTh6Vw72WzKa2q45fL8zyuTKTnvZJbxKjBA7jzoum8vKmIu5/Ppa0/NGvqm7j9bznc/Xwu50wfyjPfPqnL4QFw+Qmjuf38qTy3fjf/yN7NGRp9HnH0vxGGmZmpXHvqeJ58r4BVW0v9Lkek21TVNfLWllLOnT6Mb84bx7fmjWPJqu386c1tLW6/On8fC+5byVNrC/jOGRP5/RXHhXXepDWLTp/AotMnALBgxrBue17pHjqJHqbvnT2Zlzbt5QfLcnjpptMYkNDx48QikWrlRyXUNwY4d3rwUtkfXjCNPRW1/GJ5HkNT+3PxrBEAVNc38j8vfciSVdsZNXgAS6+dy0kTvDk3cduCKXzpuJFMHJLsyfNL+NQDCdOAhFh+eelMduyrZsljr8H110NKCsTEBP+9/nrYutXvMkU65ZVNexk8MIGssYMBiIkx7vnyLE4YO5j/fHo97+bvY9XWUs77zUqWrNrO1SeP5eXvneZZeACYmcIjQqkH0gUnT0jnztgdfPXbtxBwAWIaG4IrKivhT3+CRx6BZ56B88/3t1CRDmhoCrBiczHnzxh22Enz/qErqr704Dtc/fAaahsCjD0qkaf/4yROGDfYx4rFb571QMzsITMrNrONraxPNbPnzGy9mW0ys4XN1i0wsw/NbIuZ3e5VjV22dSsLf3MriQ11n4bHIQ0NUF0NX/qSeiISFVbn76eytpFzp3/2XENaYgJLFh7P+PQkrpk3jhdvOk3hIZ4ewloCLGhj/Q1ArnNuFjAfuMfMEswsFngAOB+YDlxuZtM9rDN899yDNTS0vU1DA/z61z1Tj0gXvJK7lwHxscyblN7i+sxBiSy/6VR+fNF0nfMTwMMAcc6tBPa3tQmQbMGJcZJC2zYCJwBbnHP5zrl64EngEq/q7JLHHw8GRFsaGuCxx3qmHpEwOed4ZVMRp0/O6NTAQenb/DyJfj8wDdgN5AA3OecCwEigoNl2haFlLTKz68xsrZmtLSkp8bLez6qq6t7tRHyyobCcvRW1nHu0JiqUjvMzQM4DsoERwGzgfjNLAVoa8trqKCbn3GLnXJZzLisjo4enFklK6t7tRHzySu5eYmOMM6dqqnTpOD8DZCGwzAVtAbYBUwn2OEY12y6TYC8l8lx5JcS3M4I9Ph6uuqpn6hEJ0yubijhx3GBNEiqd4meA7ATOAjCzocAUIB94D5hkZuPMLAH4KvCsb1W25fvf71iA3Hxzz9QjEob8kio+Lq76ZPCgSEd5eRnvUuAdYIqZFZrZNWa2yMwWhTa5GzjZzHKAFcBtzrlS51wj8B3gZSAPeNo5t8mrOrtkwoTgOI/ExM8ESX1MLI39BwTXT5jgU4Ei7Xs1N3ijpnOO1lQh0jmeDSR0zl3ezvrdwLmtrFsOLPeirm53/vmwYUPwUt3HHoOqKlxSEq/OOps/HHcxD596Jpp8WiLZK7lFzBiZopulSadpKpPuMGEC3H8/lJdDUxNWXs7Ep5eQl5jBz57P9bs66SLnHJt2l/Ph3kq/S+l2xZW1vL/zQIuDB0Xao6lMPDJlWDLXz5/IfSs+5pLZIzhzqo4vR5KDdY2UVNYxIm3AYXfDOyQQcHxQcICXNu7lpU17KdhfA8D8KRnceOYkjhsT3o2SIs2KvGKcQ5fvSlgUIB66/owJLM/Zwx1/38hj1wxkfPpAYtq5MY94xznHuh0HePK9Al7YsIeahibMYHhKfzIHJzJ6cCKjBiVSWlXHy5v2UlxZR3ysMW9iOt85YyKlVfX86c18vvj7VcybmM6NZ06M6rvj1TY0sTxnD6MHJzJlqCYrlM7THQk99v7OA3z5wXdoCjgSE2KZNjyFo0ekMH14CkePSGXGyBSCg/GjXyDgeH/nAdbuOEBjU4CmAAScwzlHk3PEmJE6IJ5BiQmkJcaTlpjAoMR4MpL7kdzBG3qFo6SyjmXvF/LU2gLySw4yMCGWz80awbGjB7GrrIaCA9UU7K+mYH8Neytq6R8fw/zJQzh/5jDOmDrksJuNHaxr5C+rd7B4ZT6lVfWcOG4wt5wzOeKDpKEpwEdFleQUlrO+sJycXWVs3lNJY8Bx/fwJ/NeCqX6XKD7RLW1DIjFAALaVHuS9bfvJ3VNB7u4KcvdUUFXXCEDWmEH87JIZTB+R0ur+dY1NPLpqB39ZvYObz5nMJbNbHZjf45oCjve27+fFnD28uDH4V/uRzCDGLBQmn30Os+CdHk8cN5gTxw3mhHGDOSqp32dep6Syjt3lNewpq2VPeQ27yoLf7y6vYXdZLRU1DcTGGHExRlysERsTQ1yMUVpVR2PAkTVmEJcdP4oLZw5nYL+WO9+1DU0A7U7nUVPfxNI1O3nw31sprqzj4lkjuOPCaQxN6d/BlvvUnvIafvyPTYxM68+dnzu63dvHdoZzjuU5e/nJc5soCf3fpPSP45jMNI7JTOWYzFTOnjZUd/rrwxQgIZEaIEcKBBwFB6pZ+XEpv371I8qq6/n6SWO5+ZzJh91j3TnHixv38qsXN7NzfzUZyf0orarjV5fO5CvHj/at/vKaBlbn7+PfH5Xw8qa9lFbV0y8uhjOmBP9qnz95CIn9YokxI8b4pIcVCDgqaxspq6nnQHUDB6rrKa9uYMe+atZs38e6HQeobQgAMHFIEhMzkiiurGVveS1FlXU0BQ7/rA5MiGV42gBGpA1gRGp/0hITCDhHY5OjKRCgIeBoanKkJyfwhTmZTBzS/TMC1DY08bs3tvLgv7cSH2PcfM5kvnHyWOI7+Av5pY17uO1vOdTUN1HfFOCCmcP4zVfmtHheprOKKmr50T828mroKqtrTx3PrMw0xhyV2Gt6vdJ1CpCQaAmQ5sqq67nnlY94fPUOjhqYwO3nT+PSOSPZsKucnz+fy9odB5gyNJk7LslDBnAAAAskSURBVJzG8WMHs+jxdfz7oxLu+tx0Fp4yrkdqrKlvYu2O/azauo9VW0rJ2VVOwMGA+FjOnDqEC2YOZ/6UjFb/qu+o+sYAObvKWbNtP6u37aNgfzXDUvszLGUAw1P7Mzytf/Df1GBopPSPi5hfhDv2HeQnz27i9Q9LmDw0iZ9ePKPNmywdrGvkZ8/l8tTaAo7JTOW+r85hRV4RP38hj9MnZ/Dglce1OuPt/oP1/PHNfAIBx9zxR5E1dtBhhwADAceT7xXwy+V51DcFuOWcyVwzb5x6GdIiBUhINAbIIRt3lfPjf27kg51ljD0qke37qklP6sf3z53MZVmjPjmsUdfYxHeXfsDLm4q49bwp3HDGxC6/dmVtAw+/vZ1l7xdS09BEUwCaAgGaAo6mgKO2Mfh9XIwxZ3QaJ01I5+QJRzFndBr94jRz6yHOOV7LK+anz22i8EANJ4wbzJxRaczMTOWYkWmMGjwAM2NDYRk3PZnN9n0H+fbpE7j5nMmf9Fieem8nty/L4fgxg/nT1VmHnX+pbwzw6DvbuW/Fx1TXNxFrRn1TgNgYY8bIVOaOH8zszDQeeWc77+bv56TxR/HLS2cyNn2gPw0iUUEBEhLNAQLBvxyfeb+QR9/ZzvzJQ1g0fwJJLfxV39gU4NZnNvD3D3Zx/fwJ3HrelE/+Ej9wsJ538vfx9pZS8vZUcPzYwVx4zHBmjkz9zF/rB+saWbJqO398M5+y6gZOm5zB8JT+xMYGzyPEWPDfxIRYjh0ziOPHDu5yL6MvqG1oYvHKfF7LK2Lznkrqm4KH5VIHxDN1WDLrdhwgI7kf9142u8VeyvMbdnPzU9lMGZbMIwtPYPDABFbkFfOL5XlsKz3IaZMz+PGF08gclMgHOw/wTv4+3s3fR3ZBGQ1NjuT+cdxxwTS+cvyoiOmhSeRSgIREe4B0RiDguOMfG1m6ZidfPX4UaYkJvL2llI27y3EOkvrFMXFIEht3ldMYcIwaPIALZg7nwpnDmTQkmcfe3c6D/85n/8F6zpw6hJvPnszMzFS/31avU98YvPppQ+jKp427Kpg0NIk7L5re5sSFr39YzKLH1pE5KHi47s2PS5mQMZAfXTSdM6a0PGNudX0jOYXljM9IIiO5X4vbiBxJARLSlwIEgodMfv5CHn9+axtxMcaxowdxysR05k06imMy04iPjaGsup5Xcot4YcMe3t5SSmPAkRAbQ31TgFMnpXPzOZM5dnTvGBTX26zO38c1j6wlNsb43tmTuHLumA6fnBfpKAVISF8LEAiGSH7pQYal9G/38FJZdT2vbCpifWEZl8weqXtaR4Hiilr6xccednWeSHdSgIT0xQAREemKrgSI+sMiIhIWBYiIiIRFASIiImFRgIiISFgUICIiEhYFiIiIhEUBIiIiYVGAiIhIWHrVQEIzKwc+bmV1KlDeweVHLmvvcTpQ2qliO6a1mrtjn7a286qtvGqn1mrrjn28aqcjl0XyZ6qj+/nxmYLIaquutlNb6736+RvjnMtoq+BWudAtR3vDF7C4s+taWn7ksg48XtvT76er+/jRVl61k5dt5VU7tdA2EfuZ8rutetvPX3vbRNPPX287hPVcGOtaWn7ksvYeeyWc1+noPmqrju3jVTsduSyS26mj++kz1fV2amt9xLVVrzqE5RczW+vCnEumL1E7dZzaquPUVh3jRTv1th6IXxb7XUCUUDt1nNqq49RWHdPt7aQeiIiIhEU9EBERCYsCREREwqIAacbMHjKzYjPbGMa+x5lZjpltMbPfmpk1W3eZmeWa2SYze6J7q/aHF21lZlebWYmZZYe+vtX9lfc8rz5XofVfMjNnZlF/Etmjz9Si0PJsM3vLzKZ3f+U9z6O2uiX0e2qDma0wszHtPpkX109H6xdwGnAssDGMfdcAJwEGvAicH1o+CfgAGBR6PMTv9xnBbXU1cL/f7y0a2iq0LhlYCbwLZPn9PiOxnYCUZttcDLzk9/uM4LY6A0gMff9t4Kn2nks9kGaccyuB/c2XmdkEM3vJzNaZ2ZtmNvXI/cxsOMEP6jsu2PqPAp8Prb4WeMA5dyD0GsXevoue4VFb9UoettXdwP8AtR6W32O8aCfnXEWzTQcCveKqIY/a6nXnXHVo03eBzPbqUIC0bzFwo3PuOOA/gd+1sM1IoLDZ48LQMoDJwGQze9vM3jWzBZ5W66+uthXAF0Nd6GfMbJR3pfquS21lZnOAUc65570u1Gdd/kyZ2Q1mtpVg2H7Xw1r91h0/f4dcQ7B30qa4MIrsM8wsCTgZ+GuzQ8/9Wtq0hWWH/tKJI3gYaz7BRH/TzGY458q6t1p/dVNbPQcsdc7Vmdki4BHgzO6u1W9dbSsziwF+TfCQX6/VTZ8pnHMPAA+Y2deAHwHf6OZSfdddbRV6riuBLOD09l5XAdK2GKDMOTe7+UIziwXWhR4+C/yew7t7mcDu0PeFwLvOuQZgm5l9SDBQ3vOycB90ua2cc/uaLf8j8H88q9ZfXW2rZGAG8Ebol8Uw4Fkzu9g5t9bj2ntSd/z8NfdkaNveqFvayszOBu4ATnfO1bX7qn6fDIq0L2AszU5MAauAL4e+N2BWK/u9B8zl0xNTF4SWLwAeCX2fDhQAR/n9PiO0rYY32+YLBIPX9/cZiW11xDZv0AtOonv0mZrUbJvP4eGEnr2greYAW5u3Wbs1+N0IkfQFLAX2AA0Eew7XAOOAl4D1QC5wZyv7ZgEbQ/8B9/PpKH8D7g3tmwN81e/3GcFt9UtgU2j/14Gpfr/PSG2rI7bpFQHi0WfqvtBnKjv0mTra7/cZwW31GlAUaqts4Nn26tBUJiIiEhZdhSUiImFRgIiISFgUICIiEhYFiIiIhEUBIiIiYVGASK9mZlU9/Hqruul55ptZuZl9YGabzex/O7DP53vLbLMSHRQgIp1gZm3O3uCcO7kbX+5N59wcggO8LjKzU9rZ/vOAAkR6jKYykT7HzCYADwAZQDVwrXNus5l9juBcSQnAPuAK51yRmf0EGEFw5G+pmX0EjAbGh/79jXPut6HnrnLOJZnZfOAnQCnBaUfWAVc655yZXUBwcGkp8D4w3jl3UWv1OudqzCybTydSvBa4LlTnFuAqYDbB6cpPN7MfAV8M7f6Z99mFphM5jHog0he1NmvpW8Dc0F/9TwL/1Wyf44BLnHNfCz2eCpwHnADcZWbxLbzOHOB7BHsF44FTzKw/8AeC92CYR/CXe5vMbBDB+dNWhhYtc84d75ybBeQB1zjnVhGc6+hW59xs59zWNt6nSLdQD0T6lHZmLc0EngrdMyEB2NZs12edczXNHr/ggpPN1ZlZMTCUw6fJBljjnCsMvW42wR5MFZDvnDv03EsJ9iZacqqZbQCmAL9yzu0NLZ9hZj8H0oAk4OVOvk+RbqEAkb6mxVlLQ/4fcK9z7tlmh6AOOXjEts1nKm2i5Z+llrZpaTrt1rzpnLvIzCYDb5nZ351z2cAS4PPOufVmdjXBWwUcqa33KdItdAhL+hQXvEPdNjP7MoAFzQqtTgV2hb736p4Rm4HxZjY29Pgr7e3gnPuI4ESTt4UWJQN7QofNrmi2aWVoXXvvU6RbKECkt0s0s8JmX7cQ/KV7jZmtJzhT6yWhbX9C8JDPmwRPcHe70GGw64GXzOwtgrOflndg1weB08xsHPBjYDXwKsFAOuRJ4NbQpb8TaP19inQLzcYr0sPMLMk5V2XBkxMPAB87537td10inaUeiEjPuzZ0Un0TwcNmf/C5HpGwqAciIiJhUQ9ERETCogAREZGwKEBERCQsChAREQmLAkRERMLy/wFL0BNJcgyiMQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.recorder.plot(skip_end=10,suggestion=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.785770</td>\n",
       "      <td>1.771890</td>\n",
       "      <td>0.202346</td>\n",
       "      <td>0.797654</td>\n",
       "      <td>00:18</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>1.777209</td>\n",
       "      <td>1.767872</td>\n",
       "      <td>0.201369</td>\n",
       "      <td>0.798631</td>\n",
       "      <td>00:17</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXoAAAD4CAYAAADiry33AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAcWUlEQVR4nO3de5Ccdb3n8fenb3NPJpcBcgESLUpBNkYcIy4UxuPqJgjGs1JWKDyepbRSXiiVLc+aU6fKy65/6J6trS3PIlTUrOsuhrJAhD2FgO7K5tRBlInGEMIthECGAJlcJslkZrqnu7/7Rz8JzTCT6SSdZPLweVV19fP8Lt2/X2f49NPPDUUEZmaWXpmzPQAzMzu9HPRmZinnoDczSzkHvZlZyjnozcxSLne2BzCRGd2z46KLL6ZcDdryWbIZne0hmZlNa5s2bdobET0T1U3LoC+2zWFo5XcAmNHVQjYj9g2VeFtPB28/r5OxcpULZrby9p5Oli2eTUbi7T0d5LKv/0ApliuMlqocGh1j35ESWYliuQJAeyHHaLnC/qESna05du0fpliuMlKqUKpUuWzeDIZLFV7cf4RCNkNLLgMSM1pzXDi7neFihd2DI7xzXhdv7+nkwHCJoWKZ7rYCuw+OcNHsduZ2tpyVz87M3pokvThZ3bQM+nqHRscYHavS3Z4nn83wyNN7OH9GK//v2QGK5eqxdoVchkI2w9t6OgB46pVDjFXO3jUCczoKtOazFMsVZrUXaMln2HOoSC4jOltzdLTkeGnfMNUIypVgfncbmYwYq1SZN7MVSSya087lC2bSksvQ09VCVuLwaJlsRrw8OMLgcIlKFQZHSpQrQalc5YPv6GFOR4GOlhxthSzDxQrFcu0LDKCns4XXDhUpV6vMaMuTz2SQoFwN5nQUKFWqlMpVDhwpcWB4jGoE2/cMsedwkYygGnDh7Da6WvOUK1U6W3KUq0FWYuHsNma25RkpVZjb2UIlgq6WHD1dLUjT71fZodExWnNZCrnaBkJEUA2O/YKsVoPRcoVDI2UGR0p0FHJ0t+fpbMlNy/mYTUbT8YKpBZdcHtu2/IlCNkNbIUu1GpQqVVrz2WNthktlfr9jPzv2HqFSrdbCq1Kl78UDSLD0wm7mzWzjvK4WZncUqEatD0ClGuSyGXo6W6hGMLezhbZ8llkdeQJ48uVDdLXmuHhOO0eKFUbGKsd+EezYe4S2fJbu9jwv7D1C/4ER5nYWaC/keO3QKDNa8+wfLvHS/mFGSxVa8hn2Hynx2qEiszsKzO4ocHBkjMOjYyye20kuIzKCF/YNM1Iq091e4OUDI2QysGPgCMOlypSfVyGXgeBYmDebVPviioBKBIPDYyfUv72Q5X2LZjOjLc+zrx4mmxEt+Qy5jCjkMoyUKhwaLbNvqMjCWe3ks7UQbcllyWSgkM2Qy2YYGi1TiaA4ViGS1+3paqWjkGV2R4HdgyOMjFWYN7ONsUqVYrnK4PAYg8MlBkfGKJWr7Bsqks2ICDhcrP09dLfnmd1e4MBwiUOjZWa05hguVd6wIVGvkMswt6PA3K4WCtkMQ8UyrfksXa05cpnal3F3e56MRCX52wV4/+LZtS90ifZC9lhduRK1L/xq0NGSo7MlS0ehtjFw9G9zZnv+5P8Bz3EDh4tUqkEhl6G7LY9Ew1+0B4fHaCu8/mWeRkPFMnsPF1nc07kpInonajMtg763tzf6+vrO9jDOunKlysuDI4xVquw5VKRYrpLJiK7WHD2dtS+wXFa05GpfgMVyhS39BymVqwwVy4yUKnS01MJnuFQhmxGDwyXmdbfRUchycGSMasBYpUpLLsOrh0ZpL2QpZGtferPaa+G+aG477YXXf/y9enCUIGjNZRkqlsllRXGsygt7j3BguERHS44jxTIZidcOjbJz3zB/eukAI2MVzu9qZUZbjsOjtZAtV4P2Qpa2fC2sXx4coVINxpJwrAaUytXkeE0t8DOCfLb2BbHvSIl9Q0WGimVmtReY1VHglcERJNHZkmN2R4Hu9jwz22pBObezhUwSFN3teXIZ8dL+YYaKZbpa8nS35xkqlulszdGWr42rszVHd1uB85/bQObwywyXqsmjTCUgn8smv4SObpDkKJarBBAhMtkM1Wqw53CpVkYtpI4u168zbr22LLrackgZMqp9Oc5sKzCro4VqUPv8y1EbSzbDzLY8VcQrB0cp5LJ0teZpK2Rr4agMAoqV4ODIGP0HRjk0Wk5+gUFnS47O1jylSjAyVqUtn6UStX+nVw+XaM1l6O4ocKRYpRrQVsgyMlbl0GiZrtYCc7sKjIxVWTSngwtmtjNWrVANUaoEo6UKzw0coau1wEjyZV2N2pdhLpehlMyhWK693lCxzHCpyt6h0rHPRAKUIZvJ0NGSZXZnK12tOQrZLIV8luFShVcOFilXa1+crx4qEoi2fIbO1gKdLXlGy7VduvNntXPejDaGi2Xy2QzFSpBRhgWz2msbfSGOftVHQDabZX53bSMil81ypFQhn639an/tUJHnBobZd6RET2cLF83poBKwb6jEaKXCy4Oj7D8yxpzOFg6PlHltqMiM1gIz2vK0t+RoK+RpyWWY0ZqnVK2No7u9wPzuVuZ3dzCrs7aBKMQjzw6w7dUhKgFdhRyb+w9ypFTh2e/9m5MPeknrgeuAPRFx+QT1fwPclKzmgEuBnojYL2kncBioAOXJBjGeg95O1FilSlYiczoP3P/4o7DrDxyNZJs+xgrd9F/xdUZnvg1I8261oPXgDhb+8XvkS4NvqNG3D51S0F8DDAE/nSjox7W9Hrg1Iv4iWd8J9EbE3obngYPeziERtUdtZYLleL3dSdXzxvq65Wq1SkZ6w2tFBEPFMlEN2gtCEuVKldFSmYhIjkMELTnRUci+HolNHHdE8MrgCPuOFMmqthXeUciSy4jzugpEtda2VhfjxnCin1Ft6YXiDLrmXMCc7hlv3K0Tb1oY583lEfGG0qOvVjueVvvMIwJJBEEGIUH22PsGR0f9xh1Gk4whJl6J5D0r1dojq9r64aFhhva8yOKRJ6j/jHTN1yYN+ikPxkbERkmLpmqXuBHY0GBbs3OfkiQ7Cyba6yyga1xZFjiT54AJmL8A5p/B9xx96ikWXXBhUw6Si4l/E2STx6m8xomOY6L3bOkO9g6NwXs/PK7ma5O+VtOOUEhqB1YA99QVB/CwpE2S1kzRf42kPkl9AwMDzRqWmb1FvFXOhDqZeTbzUPT1wD9HxP66sqsi4gpgJfClZDfQhCJiXUT0RkRvT8+E5/ybmdlJaGbQr2bcbpuI2J087wHuBZY18f3MzKaFwcFBfvCDH5xwv2uvvZbBwcGpG56ipgS9pJnAB4H76so6JHUdXQY+CmxtxvuZmU0nkwV9pXL862AeeOABuru7T9ewjpnyYKykDcByYK6kfuCbQB4gIu5Imv0l8HBEHKnrej5wb7I/KQf8LCIebN7Qzcymh7Vr1/L888+zdOlS8vk8nZ2dzJs3j82bN7Nt2zY+8YlPsGvXLkZHR/nKV77CmjW1Q5aLFi2ir6+PoaEhVq5cydVXX82jjz7KggULuO+++2hra2vK+Bo56+bGBtr8BPjJuLIdwLtPdmBmZifj2//7SbbtPtTU17xs/gy+ef27Jq3/7ne/y9atW9m8eTOPPPIIH/vYx9i6dSuLFy8GYP369cyePZuRkRHe97738clPfpI5c+a84TWee+45NmzYwA9/+EM+9alPcc899/DpT3+6KeOf9ve6MTM71yxbtuxYyAN8//vf59577wVg165dPPfcc28K+sWLF7N06VIA3vve97Jz586mjcdBb2apcrwt7zOlo6Pj2PIjjzzCb37zG373u9/R3t7O8uXLGR0dfVOflpbXr3bIZrOMjIw0bTzpvdOPmdkZ0tXVxeHDhyesO3jwILNmzaK9vZ2nn36axx577AyPzlv0ZmanbM6cOVx11VVcfvnltLW1cf755x+rW7FiBXfccQdLlizhHe94B1deeeUZH5/vXmlm57ynnnqKSy+99GwP44yZaL6SJr3XjXfdmJmlnIPezCzlHPRmZinnoDczSzkHvZlZyjnozcxSzkFvZnYWdHZ2ArB7925uuOGGCdssX76cZpxq7qA3MzuL5s+fz913331a38NXxpqZNcHXv/51Lr74Yr74xS8C8K1vfQtJbNy4kQMHDjA2NsZ3vvMdVq1a9YZ+O3fu5LrrrmPr1q2MjIxw8803s23bNi699NKm3e/GQW9m6fKrtfDqE819zQv+Baz87nGbrF69mq9+9avHgv7nP/85Dz74ILfeeiszZsxg7969XHnllXz84x+f9P/7evvtt9Pe3s6WLVvYsmULV1xxRVOG76A3M2uC97znPezZs4fdu3czMDDArFmzmDdvHrfeeisbN24kk8nw8ssv89prr3HBBRdM+BobN27ky1/+MgBLlixhyZIlTRmbg97M0mWKLe/T6YYbbuDuu+/m1VdfZfXq1dx5550MDAywadMm8vk8ixYtmvAWxfUm29o/FT4Ya2bWJKtXr+auu+7i7rvv5oYbbuDgwYOcd9555PN5fvvb3/Liiy8et/8111zDnXfeCcDWrVvZsmVLU8blLXozsyZ517vexeHDh1mwYAHz5s3jpptu4vrrr6e3t5elS5fyzne+87j9v/CFL3DzzTezZMkSli5dyrJly5oyLt+m2MzOeb5NsW9TbGb2ljZl0EtaL2mPpK2T1C+XdFDS5uTxjbq6FZKekbRd0tpmDtzMzBrTyBb9T4AVU7T5p4hYmjz+A4CkLHAbsBK4DLhR0mWnMlgzs8lMx93Qp8PJzHPKoI+IjcD+kxjPMmB7ROyIiBJwF7Bqij5mZiestbWVffv2pT7sI4J9+/bR2tp6Qv2addbNByT9GdgNfC0ingQWALvq2vQD75/sBSStAdYAXHTRRU0alpm9FSxcuJD+/n4GBgbO9lBOu9bWVhYuXHhCfZoR9H8ELo6IIUnXAr8ELgEmOut/0q/biFgHrIPaWTdNGJeZvUXk83kWL158tocxbZ3yWTcRcSgihpLlB4C8pLnUtuAvrGu6kNoWv5mZnUGnHPSSLlByza6kZclr7gMeBy6RtFhSAVgN3H+q72dmZidmyl03kjYAy4G5kvqBbwJ5gIi4A7gB+IKkMjACrI7aEZGypFuAh4AssD7Zd29mZmeQr4w1M0sBXxlrZvYW5qA3M0s5B72ZWco56M3MUs5Bb2aWcg56M7OUc9CbmaWcg97MLOUc9GZmKeegNzNLOQe9mVnKOejNzFLOQW9mlnIOejOzlHPQm5mlnIPezCzlHPRmZinnoDczSzkHvZlZyjnozcxSzkFvZpZyUwa9pPWS9kjaOkn9TZK2JI9HJb27rm6npCckbZbU18yBm5lZYxrZov8JsOI49S8AH4yIJcB/BNaNq/9QRCyNiN6TG6KZmZ2K3FQNImKjpEXHqX+0bvUxYOGpD8vMzJql2fvoPwv8qm49gIclbZK05ngdJa2R1Cepb2BgoMnDMjN765pyi75Rkj5ELeivriu+KiJ2SzoP+LWkpyNi40T9I2IdyW6f3t7eaNa4zMze6pqyRS9pCfAjYFVE7DtaHhG7k+c9wL3Asma8n5mZNe6Ug17SRcAvgL+KiGfryjskdR1dBj4KTHjmjpmZnT5T7rqRtAFYDsyV1A98E8gDRMQdwDeAOcAPJAGUkzNszgfuTcpywM8i4sHTMAczMzuORs66uXGK+s8Bn5ugfAfw7jf3MDOzM8lXxpqZpZyD3sws5Rz0ZmYp56A3M0s5B72ZWco56M3MUs5Bb2aWcg56M7OUc9CbmaWcg97MLOUc9GZmKeegNzNLOQe9mVnKOejNzFLOQW9mlnIOejOzlHPQm5mlnIPezCzlHPRmZinnoDczSzkHvZlZyk0Z9JLWS9ojaesk9ZL0fUnbJW2RdEVd3QpJzyR1a5s5cDMza0wjW/Q/AVYcp34lcEnyWAPcDiApC9yW1F8G3CjpslMZrJmZnbgpgz4iNgL7j9NkFfDTqHkM6JY0D1gGbI+IHRFRAu5K2pqZ2RnUjH30C4Bddev9Sdlk5ROStEZSn6S+gYGBJgzLzMygOUGvCcriOOUTioh1EdEbEb09PT1NGJaZmQHkmvAa/cCFdesLgd1AYZJyMzM7g5qxRX8/8Jnk7JsrgYMR8QrwOHCJpMWSCsDqpK2ZmZ1BU27RS9oALAfmSuoHvgnkASLiDuAB4FpgOzAM3JzUlSXdAjwEZIH1EfHkaZiDmZkdx5RBHxE3TlEfwJcmqXuA2heBmZmdJb4y1sws5Rz0ZmYp56A3M0s5B72ZWco56M3MUs5Bb2aWcg56M7OUc9CbmaWcg97MLOUc9GZmKeegNzNLOQe9mVnKOejNzFLOQW9mlnIOejOzlHPQm5mlnIPezCzlHPRmZinnoDczSzkHvZlZyjnozcxSrqGgl7RC0jOStktaO0H930janDy2SqpImp3U7ZT0RFLX1+wJmJnZ8eWmaiApC9wGfAToBx6XdH9EbDvaJiL+Hvj7pP31wK0Rsb/uZT4UEXubOnIzM2tII1v0y4DtEbEjIkrAXcCq47S/EdjQjMGZmdmpayToFwC76tb7k7I3kdQOrADuqSsO4GFJmyStmexNJK2R1Cepb2BgoIFhmZlZIxoJek1QFpO0vR7453G7ba6KiCuAlcCXJF0zUceIWBcRvRHR29PT08CwzMysEY0EfT9wYd36QmD3JG1XM263TUTsTp73APdS2xVkZmZnSCNB/zhwiaTFkgrUwvz+8Y0kzQQ+CNxXV9YhqevoMvBRYGszBm5mZo2Z8qybiChLugV4CMgC6yPiSUmfT+rvSJr+JfBwRByp634+cK+ko+/1s4h4sJkTMDOz41PEZLvbz57e3t7o6/Mp92ZmjZK0KSJ6J6rzlbFmZinnoDczSzkHvZlZyjnozcxSzkFvZpZyDnozs5Rz0JuZpZyD3sws5Rz0ZmYp56A3M0s5B72ZWco56M3MUs5Bb2aWcg56M7OUc9CbmaWcg97MLOUc9GZmKeegNzNLOQe9mVnKOejNzFLOQW9mlnINBb2kFZKekbRd0toJ6pdLOihpc/L4RqN9zczs9MpN1UBSFrgN+AjQDzwu6f6I2Dau6T9FxHUn2dfMzE6TRrbolwHbI2JHRJSAu4BVDb7+qfQ1M7MmaCToFwC76tb7k7LxPiDpz5J+JeldJ9gXSWsk9UnqGxgYaGBYZmbWiEaCXhOUxbj1PwIXR8S7gX8AfnkCfWuFEesiojcient6ehoYlpmZNaKRoO8HLqxbXwjsrm8QEYciYihZfgDIS5rbSF8zMzu9Ggn6x4FLJC2WVABWA/fXN5B0gSQly8uS193XSF8zMzu9pjzrJiLKkm4BHgKywPqIeFLS55P6O4AbgC9IKgMjwOqICGDCvqdpLmZmNgHV8nh66e3tjb6+vrM9DDOzc4akTRHRO1Gdr4w1M0s5B72ZWco56M3MUs5Bb2aWcg56M7OUc9CbmaWcg97MLOUc9GZmKeegNzNLOQe9mVnKOejNzFLOQW9mlnIOejOzlHPQm5mlnIPezCzlHPRmZinnoDczSzkHvZlZyjnozcxSzkFvZpZyDnozs5RrKOglrZD0jKTtktZOUH+TpC3J41FJ766r2ynpCUmbJfU1c/BmZja13FQNJGWB24CPAP3A45Luj4htdc1eAD4YEQckrQTWAe+vq/9QROxt4rjNzKxBjWzRLwO2R8SOiCgBdwGr6htExKMRcSBZfQxY2NxhmpnZyWok6BcAu+rW+5OyyXwW+FXdegAPS9okac1knSStkdQnqW9gYKCBYZmZWSOm3HUDaIKymLCh9CFqQX91XfFVEbFb0nnAryU9HREb3/SCEeuo7fKht7d3wtc3M7MT18gWfT9wYd36QmD3+EaSlgA/AlZFxL6j5RGxO3neA9xLbVeQmZmdIY0E/ePAJZIWSyoAq4H76xtIugj4BfBXEfFsXXmHpK6jy8BHga3NGryZmU1tyl03EVGWdAvwEJAF1kfEk5I+n9TfAXwDmAP8QBJAOSJ6gfOBe5OyHPCziHjwtMzEzMwmpIjptzu8t7c3+vp8yr2ZWaMkbUo2sN/EV8aamaWcg97MLOUc9GZmKeegNzNLOQe9mVnKOejNzFLOQW9mlnIOejOzlHPQm5mlnIPezCzlHPRmZinnoDczSzkHvZlZyjnozcxSzkFvZpZyDnozs5Rz0JuZpZyD3sws5Rz0ZmYp56A3M0s5B72ZWco1FPSSVkh6RtJ2SWsnqJek7yf1WyRd0WhfMzM7vaYMeklZ4DZgJXAZcKOky8Y1WwlckjzWALefQF8zMzuNGtmiXwZsj4gdEVEC7gJWjWuzCvhp1DwGdEua12BfMzM7jXINtFkA7Kpb7wfe30CbBQ32BUDSGmq/BgCKkrY2MLZz3Vxg79kexBngeabLW2WecG7N9eLJKhoJek1QFg22aaRvrTBiHbAOQFJfRPQ2MLZzmueZLp5n+qRlro0EfT9wYd36QmB3g20KDfQ1M7PTqJF99I8Dl0haLKkArAbuH9fmfuAzydk3VwIHI+KVBvuamdlpNOUWfUSUJd0CPARkgfUR8aSkzyf1dwAPANcC24Fh4Obj9W1gXOtOZjLnIM8zXTzP9EnFXBUx4S5zMzNLCV8Za2aWcg56M7OUm1ZBn6bbJUi6UNJvJT0l6UlJX0nKZ0v6taTnkudZdX3+Npn7M5L+9dkb/YmTlJX0J0n/mKynbp6SuiXdLenp5N/1A2mcJ4CkW5O/262SNkhqTcNcJa2XtKf+Op2TmZek90p6Iqn7vqSJTiWfPiJiWjyoHax9HngbtdMy/wxcdrbHdQrzmQdckSx3Ac9Suw3EfwLWJuVrge8ly5clc24BFiefRfZsz+ME5vvvgJ8B/5isp26ewP8APpcsF4DulM5zAfAC0Jas/xz4t2mYK3ANcAWwta7shOcF/AH4ALVrhX4FrDzbczveYzpt0afqdgkR8UpE/DFZPgw8Re0/oFXUAoPk+RPJ8irgrogoRsQL1M5gWnZmR31yJC0EPgb8qK44VfOUNINaSPwYICJKETFIyuZZJwe0ScoB7dSufznn5xoRG4H944pPaF7J7V1mRMTvopb6P63rMy1Np6Cf7DYK5zxJi4D3AL8Hzo/aNQYkz+clzc7l+f9X4N8D1bqytM3zbcAA8N+TXVQ/ktRB+uZJRLwM/GfgJeAVatfFPEwK55o40XktSJbHl09b0ynoG75dwrlEUidwD/DViDh0vKYTlE37+Uu6DtgTEZsa7TJB2bSfJ7Ut3CuA2yPiPcARaj/zJ3OuzpNkH/Uqarsr5gMdkj59vC4TlJ0Tc53CKd/aZbqYTkHfyK0WzimS8tRC/s6I+EVS/Fry04/keU9Sfq7O/yrg45J2Utvd9heS/hfpm2c/0B8Rv0/W76YW/GmbJ8C/Al6IiIGIGAN+AfxL0jlXOPF59SfL48unrekU9Km6XUJyFP7HwFMR8V/qqu4H/jpZ/mvgvrry1ZJaJC2mdm//P5yp8Z6siPjbiFgYEYuo/Zv934j4NOmb56vALknvSIo+DGwjZfNMvARcKak9+Tv+MLVjTGmcK5zgvJLdO4clXZl8Pp+p6zM9ne2jweOOiF9L7eyU54G/O9vjOcW5XE3t59wWYHPyuBaYA/wf4LnkeXZdn79L5v4M0/wo/iRzXs7rZ92kbp7AUqAv+Tf9JTArjfNMxv5t4GlgK/A/qZ15cs7PFdhA7bjDGLUt88+ezLyA3uSzeR74byR3GZiuD98Cwcws5abTrhszMzsNHPRmZinnoDczSzkHvZlZyjnozcxSzkFvZpZyDnozs5T7/x16nN915MpBAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.fit_one_cycle(2,1.10E-06,moms=(0.8,0.7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.save('first_cycle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed_all(seed)\n",
    "learner.load('first_cycle');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.freeze_to(-2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='0' class='' max='1' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      0.00% [0/1 00:00<00:00]\n",
       "    </div>\n",
       "    \n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "  </tbody>\n",
       "</table><p>\n",
       "\n",
       "    <div>\n",
       "        <style>\n",
       "            /* Turns off some styling */\n",
       "            progress {\n",
       "                /* gets rid of default border in Firefox and Opera. */\n",
       "                border: none;\n",
       "                /* Needs to be in here for Safari polyfill so background images work as expected. */\n",
       "                background-size: auto;\n",
       "            }\n",
       "            .progress-bar-interrupted, .progress-bar-interrupted::-webkit-progress-bar {\n",
       "                background: #F44336;\n",
       "            }\n",
       "        </style>\n",
       "      <progress value='65' class='' max='575' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      11.30% [65/575 00:02<00:21 3.8189]\n",
       "    </div>\n",
       "    "
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "LR Finder is complete, type {learner_name}.recorder.plot() to see the graph.\n",
      "Min numerical gradient: 2.09E-05\n",
      "Min loss divided by 10: 2.51E-06\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEGCAYAAABo25JHAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZxcZZ3v8c+vuqv3LaE7ISudRCAsAsEGQUBZ3HAUUVFHZBHBjA5XYYZxnOtr7uiM975G546oCDPIFQcdGRwH0EEZHAFBQGQJMQtJE5J0ICQd0t1Jet9q+d0/qjo0Ta9Jnzq1fN+vV9FV5zznnF89VOpXz3Oe8xxzd0REpHBFwg5ARETCpUQgIlLglAhERAqcEoGISIFTIhARKXDFYQcwU/X19d7Y2Bh2GCIiOeW5557rcPeG8dblXCJobGxkzZo1YYchIpJTzOzlidapa0hEpMApEYiIFDglAhGRAqdEICJS4JQIREQKnBKBiEiBUyIQESlwSgQiIjngOw9t5YmtHYHsW4lARCTLDcYSfPvhF3n2pf2B7F+JQEQky728rx93WN5QGcj+lQhERLJcS3svAMvrqwLZvxKBiEiWa+noA2CZWgQiIoVpe3sv82tKqSoNZp5QJQIRkSzX0t4XWLcQKBGIiGQ1d6elvTewE8WgRCAiktX29w3TPRhneYNaBCIiBWnkRLFaBCIiBWpk6OgKnSMQESlMLe19lBRFWDSnPLBjKBGIiGSx7e19HHVEBUURC+wYgSUCM1tiZo+YWbOZbTKz6yYpe5qZJczskqDiERHJRS0dwY4YgmBbBHHgBnc/DjgDuNbMjh9byMyKgG8A/x1gLCIiOSeWSLJzX3+gI4YgwETg7nvcfW36eQ/QDCwap+jngXuAtqBiERHJRa/s7yeedJbX526L4CAzawRWAU+PWb4I+BBwaybiEBHJJS3tI0NHc7RFMMLMqkj94r/e3bvHrP428CV3T0yxj9VmtsbM1rS3twcVqohIVmnpSA8dDfgcQTAzGKWZWZRUErjT3e8dp0gT8BMzA6gH3mdmcXf/+ehC7n4bcBtAU1OTBxmziEi22NHRx9zKEuoqSgI9TmCJwFLf7rcDze5+43hl3H3ZqPJ3AL8cmwRERArV9va+wM8PQLAtgrOAy4GNZrYuvezLwFIAd9d5ARGRSbS093H+yobAjxNYInD3J4BpXwHh7p8KKhYRkVzTPRijo3co8BPFoCuLRUSy0siIoWUZ6BpSIhARyUIHJ5sLeMQQKBGIiGSllvY+iiLG0rlKBCIiBamlo5clc8opKQ7+a1qJQEQkC7W092XkRDEoEYiIZJ1k0tnRkZlrCECJQEQk67R2DTAUT6pFICJSqF6bbE4tAhGRgjQydFSJQESkQLV09FFdWkxDVWlGjqdEICKSZVra+1jWUEl6ZubAKRGIiGSZlvbejI0YAiUCEZGs0j8cp7VrMGMjhkCJQEQkq+zoyOyIIVAiEBHJKgeHjtarRSAiUpAyOf30CCUCEZEssqOjl0V15ZSXFGXsmEoEIiJZpKWjL6PnB0CJQEQka7h7atbRDHYLgRKBiEjWaO8ZoncontGhoxBgIjCzJWb2iJk1m9kmM7tunDIfNLMNZrbOzNaY2dlBxSMiku22pecYyuSJYoDiAPcdB25w97VmVg08Z2YPuvvmUWUeBu5zdzezk4CfAisDjElEJGs9tX0fEYM3L6rN6HEDaxG4+x53X5t+3gM0A4vGlOl1d0+/rAQcEZEC9WBzG02Nc5lTWZLR42bkHIGZNQKrgKfHWfchM3sBuB/49ATbr053Ha1pb28PMlQRkVDsOtBP855u3nXc/IwfO/BEYGZVwD3A9e7ePXa9u//M3VcCFwNfG28f7n6buze5e1NDQ0OwAYuIhOChzXsBeOfxeZYIzCxKKgnc6e73TlbW3R8DVphZfZAxiYhko4ea21jRUJnxE8UQ7KghA24Hmt39xgnKvCldDjM7FSgB9gUVk4hINuoejPFUyz7edfyRoRw/yFFDZwGXAxvNbF162ZeBpQDufivwEeAKM4sBA8DHR508FhEpCL/d0k486bzr+HmhHD+wRODuTwCT3l7H3b8BfCOoGEREcsGDm/dyRGUJpyyZE8rxdWWxiEiIYokkj2xp4/yV8yiKZObWlGMpEYiIhOjZHfvpGYzzrhBGC41QIhARCdGDzXspLY5w9tHhDZhUIhARCYm78+DmvZz9pnoqSoIcuzM5JQIRkZBs2dvDrgMDoVxENpoSgYhISEauJr7guHCGjY5QIhARCcmDzW2csqSOedVlocahRCAiEoK27kHWv9IZ6mihEUoEIiIheKi5DYB3hjDb6FhKBCIiIXioeS9L51ZwzPzM3pZyPEoEIiIZ1j8c54ltHbzzuPmk590MlRKBiEiGPb61g+F4kneGNMncWEoEIiIZ9uiWNqrLijmtcW7YoQBKBCIiGdfS3sfKI6uJFmXHV3B2RCEiUkBauwZYVFcedhgHKRGIiGRQIuns6RxkoRKBiEhhau8ZIp50JQIRkUK1u7MfgEVzlAhERArS7s5BAJ0jEBEpVK2dAwCF0TVkZkvM7BEzazazTWZ23ThlPmlmG9KPJ83s5KDiERHJBrsPDFBbHqWqNLwb0YwVZCRx4AZ3X2tm1cBzZvagu28eVWYH8A53P2BmFwK3AW8NMCYRkVC1dg5kVWsAAmwRuPsed1+bft4DNAOLxpR50t0PpF8+BSwOKh4RkWywuzO7riGADJ0jMLNGYBXw9CTFrgYemGD71Wa2xszWtLe3z36AIiIZkkoE4d6IZqzAE4GZVQH3ANe7e/cEZc4jlQi+NN56d7/N3ZvcvamhoSG4YEVEAtQ9GKNnMJ51XUOBnq0wsyipJHCnu987QZmTgO8DF7r7viDjEREJ056RoaNZdA0BBDtqyIDbgWZ3v3GCMkuBe4HL3f3FoGIREckGIxeTFVKL4CzgcmCjma1LL/sysBTA3W8F/gY4Avin9M0Z4u7eFGBMIiKhycaLySDARODuTwCT3nrH3a8BrgkqBhGRbNLaOUC0yGioKg07lNfRlcUiIhmy+8AAC2rLiUTCvz3laEoEIiIZkrqYLLuGjoISgYhIxrR2DrCoriLsMN5AiUBEJANiiSSvdg9m3cVkoEQgIpIRe7sHSXr2XUMASgQiIhnRmh46mm3XEIASgYhIRmTrxWSgRCAikhGtWXoxGSgRiIhkxK4DAxxRWUJZtCjsUN5AiUBEJAOy8YY0I5QIREQyoDULb0gzQolARCRg7s7uXG8RmFmlmUXSz48xs4vS9xoQEZEpdA3E6B9OZOX0EjD9FsFjQJmZLQIeBq4C7ggqKBGRfLK7cwCAxVl4MRlMPxGYu/cDHwa+6+4fAo4PLiwRkfyx+0AqEeR01xCpG46dCXwSuD+9LNDbXIqI5IvWzvxIBNcD/xP4mbtvMrPlwCPBhSUikj9auwYpLY5wRGVJ2KGMa1q/6t39t8BvAdInjTvc/QtBBiYiki92H0gNHU3fkjfrTHfU0L+ZWY2ZVQKbgS1m9sVgQxMRyQ/ZPHQUpt81dLy7dwMXA/9F6gb0l0+2gZktMbNHzKzZzDaZ2XXjlFlpZr83syEz+4sZRy8ikgOy+WIymP4J32j6uoGLgZvdPWZmPsU2ceAGd19rZtXAc2b2oLtvHlVmP/CF9H5FRPLOUDxBW89QXrQIvge8BFQCj5nZUUD3ZBu4+x53X5t+3gM0A4vGlGlz92eB2AzjFhHJCa92pWcdzdJrCGD6J4tvAm4atehlMztvugcxs0ZgFfD0TIITEcl1uw8OHc3Oq4ph+ieLa83sRjNbk358k1TrYDrbVgH3ANenzzPMmJmtHjl2e3v7oexCRCQUIxeTZfM5gul2Df0A6AE+ln50A/8y1Ubp8wr3AHe6+72HGqS73+buTe7e1NDQcKi7ERHJuNbOQczgyNrsbRFM92TxCnf/yKjXf2tm6ybbwFIDZm8Hmt39xkMNUEQkl7V2DtBQVUppcfbdkGbEdBPBgJmd7e5PAJjZWcDAFNucRWqI6cZRSePLpIae4u63mtmRwBqgBkia2fW8NlRVRCTnZfs1BDD9RPBZ4EdmVpt+fQC4crIN0klj0svo3P1VYPE0YxARyTmtnQMct7Am7DAmNa1zBO6+3t1PBk4CTnL3VcD5gUYmIpLjRm5Ik80nimGGdyhz9+5R3TZ/HkA8IiJ5Y1/fMEPxJAuz+EQxHN6tKrNz9iQRkSwxMv30ojkVIUcyucNJBFNNMSEiUtBeuyFNdrcIJj1ZbGY9jP+Fb0B2d3qJiIRs5KribD9HMGkicPfqTAUiIpJvWjsHqSwporY8GnYokzqcriEREZnE7s5+FmbxDWlGKBGIiASktXMw6y8mAyUCEZHAtHYOZPX00yOUCEREAvDsS/vZ1zfMcQuy+6piUCIQEQnETQ9vpb6qhEtOzf5ZdJQIRERm2dqdB3h8awefOWc55SXZO+voCCUCEZFZ9t2HtzKnIsplZxwVdijTokQgIjKLNu7q4pEt7VxzznIqS6c7wXO4lAhERGbRTb/ZSm15lCvOzI3WACgRiIjMmk2tXTy4eS9Xn72M6rLsvpp4NCUCEZFZcvNvtlFdVsyVb2sMO5QZUSIQEZkFW17t4YHnX+WqtzVm/dxCYykRiIjMgu/+ZiuVJUV8+uxlYYcyY0oEIiKHaVtbD/dv3MOVb2ukrqIk7HBmLLBEYGZLzOwRM2s2s01mdt04ZczMbjKzbWa2wcxODSoeEZGg3PybbZRHi7jmnOVhh3JIgmwRxIEb3P044AzgWjM7fkyZC4Gj04/VwD8HGI+IyKzb0dHHfetbufyMo5hbmXutAQgwEbj7Hndfm37eAzQDi8YU+yDwI095CqgzswVBxSQiMttu/s02SoojOdsagAydIzCzRmAV8PSYVYuAV0a93sUbkwVmttrM1pjZmvb29qDCFBGZkZf39fHzdbv55FuPoqG6NOxwDlngicDMqoB7gOvdvXvs6nE2ecM9kt39NndvcvemhoaGIMIUEZmxWx7ZRnHE+JO3525rAAJOBGYWJZUE7nT3e8cpsgtYMur1YqA1yJhERGbDK/v7uXftbj5x+lLm1ZSFHc5hCXLUkAG3A83ufuMExe4DrkiPHjoD6HL3PUHFJCIyW/7p0W1EIsbnzl0RdiiHLcip8c4CLgc2mtm69LIvA0sB3P1W4L+A9wHbgH7gqgDjERGZFbsO9PMfa3Zx6VuXMj/HWwMQYCJw9ycY/xzA6DIOXBtUDCIiQfjnR7cTsfxoDYCuLBYRmZHWzgF+uuYVPtq0mAW12X9j+ulQIhARmYFbf7sdIG9aA6BEICIyba92DfKTZ17hkrcsZvGcirDDmTVKBCIi03Trb7eTdOdPz31T2KHMKiUCEZFpaOse5K5ndvLhUxexZG7+tAZAiUBEZFq+91gL8aRz7Xn51RoAJQIRkSnt7R7kx0+9zIdWLeKoIyrDDmfWKRGIiEzhlke2kUg6111wdNihBEKJQERkErsO9HPXMzv52GlL8u7cwAglAhGRSXz34W2YGZ8/P//ODYxQIhARmcBLHX3cvXYXl56+NG+uIh6PEoGIyAS+8/BWokXGn56XP1cRj0eJQERkHFv39vDzdbu58m2NzKvO/RlGJ6NEICIyjm8/tJWKaBF/8vb8bg2AEoGIyBtsbu3m/o17uPrsZcytLAk7nMApEYiIjHHjgy9SU1bM1efk9r2Ip0uJQERklPWvdPJQ815Wv305teXRsMPJCCUCEZFRvvngi8ypiPKps5aFHUrGKBGIiKQ9+9J+Hnuxnc++YwVVpUHe0j27KBGIiKR989dbaKgu5YozG8MOJaMCSwRm9gMzazOz5ydYP8fMfmZmG8zsGTM7MahYRESm8uS2Dp5q2c+1566gvKQo7HAyKsgWwR3AeydZ/2VgnbufBFwBfCfAWEREJuTu/OOvt7Cgtow/Pn1p2OFkXGCJwN0fA/ZPUuR44OF02ReARjObH1Q8IiITefTFdtbu7OTz5x9NWbSwWgMQ7jmC9cCHAczsdOAoYPF4Bc1stZmtMbM17e3tGQxRRPKdu3Pjr19kydxyPto07ldQ3gszEXwdmGNm64DPA38A4uMVdPfb3L3J3ZsaGhoyGaOI5Llfb97Lxt1dXHfBMUSLCnP8TGjjo9y9G7gKwMwM2JF+iIhkRDKZag0sr6/k4lMWhh1OaEJLf2ZWZ2Yjk3hcAzyWTg4iIhlx/8Y9bNnbw3XvPJriAm0NQIAtAjO7CzgXqDezXcBXgCiAu98KHAf8yMwSwGbg6qBiEREZK55I8q2HXuTY+dV84KTCbQ1AgInA3T8xxfrfA/l5J2gRyXr/ua6VlvY+br3sVCIRCzucUBVuW0hEClYskeQ7D2/lhIU1vOeEI8MOJ3RKBCJSUJJJ56v3bWLn/n5uePcxpMaqFLbCmVVJRArecDzJDf+xnl+sb+Wz71jBecfOCzukrKBEICIFYWA4wefufI5Ht7TzVxeu5LPvyP9bUE6XEoGI5L2ugRhX3/Esa3ce4OsffnNBzic0GSUCEclr7T1DXPGDZ9jW1sPNl57K+968IOyQso4SgYjkrVf293P57U+zt3uI2688jbcfoylqxqNEICJ5aTCW4NN3PMuB/hg/vuatvOWoOWGHlLWUCEQkL339gRfY2tbLv159upLAFHQdgYjknUe3tHHHky/x6bOWcc7R6g6aihKBiOSV/X3DfPHuDRwzv4q/fO+xYYeTE9Q1JCJ5w935q3s20NUf44dXnV6Qdxs7FGoRiEje+OmaV/j15r188T3HcvzCmrDDyRlKBCKSF17q6ONvf7GZM5cfwdVnLws7nJyiRCAiOS+eSHL9v6+jOGJ882MnF/y00jOlcwQiktMGYwm+8/BW1r3Syc2XrmJhXXnYIeUcJQIRyRnD8SQv7u1h/a5ONu7qYv2uLl7c20Mi6Xx41SLeX+B3GjtUSgQikrXiiSTrd3Xx5LYOfre9g7U7OxmOJwGoq4jy5kW1XLByBSctruW8lZpS+lApEYhIVmntHOCB51/lyW0dPL1jP71DcQCOX1DD5WccxaqldZy0qI4lc8t1U5lZEuTN638AvB9oc/cTx1lfC/wYWJqO4x/d/V+CikdEslsy6dz5zE7+/r+a6R9O0HhEBRedspCzVtRz5oojmFtZEnaIeSvIFsEdwM3AjyZYfy2w2d0/YGYNwBYzu9PdhwOMSUSy0O7OAf7y7vX8bts+zjm6nr/74Iksq68MO6yCEVgicPfHzKxxsiJAtaXadlXAfiAeVDwikn3cnZ+ueYWv/bKZpDv/50MncunpS9Xlk2FhniO4GbgPaAWqgY+7ezLEeEQkg17tGuSv7t3Ao1vaOWP5XP7vJSezZG5F2GEVpDATwXuAdcD5wArgQTN73N27xxY0s9XAaoClS3WLOZFctq2th3/9/cvc/dwukg5/e9EJXH7GUboILERhJoKrgK+7uwPbzGwHsBJ4ZmxBd78NuA2gqanJMxqliACpbpwNu7oYjCUoKY5QWlxEaTRCSVGE0miEmrLohJO8xRNJHmpu41+feonfbdtHSVGEPzppAdddcDSNOhcQujATwU7gAuBxM5sPHAu0hBiPiIyjdyjOvWt38cMnX2J7e9+kZeurSlhQW86C2jIW1pWzsK6MgeEk//7sTlq7BllYW8YX33MsHz9tCfVVpRl6BzKVIIeP3gWcC9Sb2S7gK0AUwN1vBb4G3GFmGwEDvuTuHUHFI5LPhuNJtrb1UFFSTE1ZMdVlUUqKD28qse3tvQe7cHqH4py0uJZ/uOQkFteVMxRPph+Jg887+4Zp7RqktXOAl/b18eT2fQevATj7TfV85aITuGDlPIqLNMVZtgly1NAnpljfCrw7qONLAdm+Hb75Tfjxj6G3F6qq4LLL4IYbYMWKsKMLVN9QnLue2cn3H9/Bq92Dr1tXWhyhuixKTXkxcytKqK8qpb66hCMqS6mvLqWhqoTiSIS+4Th9Qwn6huLp53FeeLWHx7d2EC0y3n/SQq448yhWLZ357R67B2MMxhLMqy6brbcsAbBUF33uaGpq8jVr1oQdhmSLBx6ASy6BWCz1GBGNph533w0XXhh4GMmks6m1m66BGMOJBMMHfzEnGY4nKSmOUFcepa6ihDkVUWorotSVlxzyr/YDfcPc8eRL/PD3L9HZH+OM5XP5+GlLcIeewTg9gzF6BuN0D8bpHoixr2+Ifb3DdPQOcaA/Num+y6NFNFSX8tG3LOaPT19KQ7W6cPKBmT3n7k3jrdMUE3KQu9M7FGdv9xBt3YMc6I+xoK6M5fWV1FXM/lWd8USSHR19dPQOMxhPMBRLMBhLMhhLMBBLYEBDdRn1VSU0VJfSUF1KVWnxa2PMt29PJYH+/jfufCQxXHIJbNgQSMvA3Vm/q4tfrG/llxta2ds9NON91JQVs7yhiuUNlaxoqGJFQxVvmlfJ0rmVFEWMgViCgeEEg7EE/cMJeofi3L9hD3c9s5OBWIJ3HT+fz527glNn8Gs9lkiyv2+Y9p4hku5UlhZTWVJMZWkRFSXFFGn0TsFRIigwg7EEL+/rZ0dHLy0dfbS09/HK/n7aeobY2z1I/3Bi3O3mVpawrL7y4GNuZcnB0SIlRRFKiiMHR5KURSOUR4soixYd/GsGL+7tYVNrN8/v7mJTazfNe7oZis/s0pGyaIR51WWsaKjkf9zzLVYND09+U41YDL71Lbj55in3PRxPsmFXJ0/v2M+zL+0nlkiysLachXXlLKorf+3kZyzB/Rv28IsNrbyyf4CSogjnHtvA+968gIV15am6GFU3pcWRVB96f4zOgeHU3/7U3709g+zo6OPJbfu4d+3uadVBccS46JSFfO4dKzh6fvX0Km6UaFGE+TVlzK9Rd42kFEzX0La2Xn69+VUqoqlfPeUlqS+pipIiykqKGBhOcKB/mAN9w+zvi6We9w/TNxQn6ZB0J+mpX4GJpJOcoN5GFvvB/4CnnySSTizhxBJJhhPJ1N94kkQSKkqKqCwtpro09cussrSY6rJiiiMRIgZmhhlEzLD0rmOJ5MH9xQ7uLxXbSIyJ5GuPPV2DtHYNMDr0edWlHHVExcEvhvk1pcyvKWNedRl1FVFaOwfY0dHH9vY+dnT0sqOj75B++Y5WXVbMCQtrOGFhLScsrOHI2jLKokWUjSSRktTzpDsdvalfru29g6m/PUO82j3E1r093P2l91I1PDDl8bymBuvqSv//cfqGE3QPxOgaiNHRO8RzLx/gmR37WbvzAIOxVGI6Zn4VlaXFtHYO0NYzxNj/3UUR46w31XPRyQt59wnzqSmLHladQGp0Tkt7L9vbe3l5Xz8Rs1QiTX9Wy6NFlJdEWHlkjebclxmbrGuoYBLBfetb+cJdf5h2+ZqyYuZWlhxsKo98GUcs9SVgWGqs0zhGFpuRKpd+HjGjpDhCtMiIFr32S9rMGBiO0zuUoHcoRt9QqgugdyhOPJHESfVBu6cSQNIdA6LFkYP7GdlncVGEoggUmRGJ2Ov+zqspPfiLfkVDFY31lVSVzrxR2DcUp3swxnC6/3t0X/hQ/LXunZEunsFYkngiyYp5VZy4sHbWZo30SASbxuc3iXHuNx6mdyjVXx5Pvn4bs9TMlqcvm8tblx3B6cvmvm6Cs+F4kr3dqdEwrV0DxBPO+SvncYSGP0oO0TkC4AMnLeDdx8+nfzhB/3CcgeFE+nnqC6u8pIi5lSXMqSihriJKVEPcJlRZWkzlISSQ2WZVVdDTM2W54YpKTllSR015MbXlUWrKotSWpx8VUU5YWEtt+cS/6EuKIyyZW6HpDyRvhf+vOUPMLNX9EC3SdLb54rLL4Pvff/1oobGiUcquupKbPrEqc3GJ5Bj97JXcdcMNqSGik4lG4c/+LDPxiOQoJQLJXStWpK4TqKh4Y0KIRlPL77477y8qEzlcSgSS2y68MHWdwOrVUFMDkUjq7+rVqeUZuJhMJNcVzKghEZFCNtmoIbUIREQKnBKBiEiBUyIQESlwSgQiIgUu504Wm1k78PIEq2uBrhmsm86y0a/rgSBvnjNZ/LOx3VTlJlo/G3UHwdZfLtXdeMvDrLvxjjfb28303+ZMlqvuprfuKHdvGLeku+fNA7htJuums2z0a2BNWPHPxnZTlZto/WzUXdD1l0t1N826yljdhV1/qrvZr7uZxpVvXUO/mOG66SybbJ+z7VCPNd3tpio30XrV3ezW3XjLw6y7wznebNSf6u7Qyx3qutfJua6hMJnZGp9gHK5MTfV36FR3h051N7V8axEE7bawA8hxqr9Dp7o7dKq7KahFICJS4NQiEBEpcEoEIiIFrmATgZn9wMzazOz5Q9j2LWa20cy2mdlNNuq+i2b2MTPbbGabzOzfZjfq7BBE3ZnZp8ys3czWpR/XzH7k4Qvqc5def4mZuZnl7YnRgD57n00vX2dmT5jZ8bMfeZYLcnxtNj+AtwOnAs8fwrbPAGeSuj3xA8CF6eVHA38A5qRfzwv7feZQ3X0KuDns95aLdZdeVw08BjwFNIX9PnOp/oCaUWUuAn4V9vvM9KNgWwTu/hiwf/QyM1thZr8ys+fM7HEzWzl2OzNbQOqD83tPfXJ+BFycXv0Z4BZ3P5A+Rluw7yIcAdVdQQiw7r4G/AMwGGD4oQui/ty9e1TRSqDgRtAUbCKYwG3A5939LcBfAP80TplFwK5Rr3ellwEcAxxjZr8zs6fM7L2BRptdDrfuAD5iZhvM7G4zWxJcqFnnsOrOzFYBS9z9l0EHmqUO+7NnZtea2XZSyfQLAcaalQrm5vVTMbMq4G3Af4zqei0dr+g4y0Z+QRST6h46F1gMPG5mJ7p75+xGm11mqe5+Adzl7kNm9lngh8D5sx1rtjncujOzCPAtUl1rBWeWPnu4+y3ALWZ2KfDXwJWzHGpWUyJ4TQTodPdTRi80syLgufTL+4B/JvUlP2Ix0Jp+vgt4yt1jwA4z20IqMTwbZOBZ4LDrzt33jVr+/4BvBBZtdjncuqsGTgQeTX8RHgncZ2YXuXsh3MpvNv7djvaTdNmCoq6htHQ/4Q4z+yiApZzs7gl3PyX9+Bt33wP0mNkZ6VEHVwD/md7Nz4Hz0sMWQlsAAANoSURBVNvXk+oqasn8u8ms2ai7dB/uiIuA5ky/jzAcbt25e5e717t7o7s3kjpZXChJYLY+e0eP2uUfAVsz/T5CF/bZ6rAewF3AHiBG6pf81cAy4FfAemAz8DcTbNsEPA9sB27mtSu0Dbgxve1G4I/Dfp85VHd/D2xKb/8IsDLs95krdTemzKPk96ihID5730l/9talP3snhP0+M/3QFBMiIgVOXUMiIgVOiUBEpMApEYiIFDglAhGRAqdEICJS4JQIJC+YWW+Gj/fkLO3nXDPrMrM/mNkLZvaP09jm4oKcIVMCo0QgMg4zm/Sqe3d/2ywe7nF3XwWsAt5vZmdNUf5iQIlAZo2mmJC8ZWYrgFuABqAf+Iy7v2BmHyA1n0wJsA/4pLvvNbOvAguBRqDDzF4ElgLL03+/7e43pffd6+5VZnYu8FWgg9RUD88Bl7m7m9n7SF1g2AGsBZa7+/snitfdB8xsHa9NJvcZYHU6zm3A5cAppK68foeZ/TXwkfTmb3ifh1F1UmDUIpB8NtGslE8AZ6R/hf8E+MtR27wF+KC7X5p+vRJ4D3A68BUzi45znFXA9aR+pS8HzjKzMuB7pOa8P5vUl/SkzGwOqbmpHksvutfdT3P3k0lNuXG1uz9Jau6cL3pq+oTtk7xPkWlRi0Dy0hSzUi4G/j09v1EJsGPUpve5+8Co1/e7+xAwZGZtwHxeP50xwDPuvit93HWkWhS9QIu7j+z7LlK/7sdzjpltAI4Fvu7ur6aXn2hm/xuoA6qA/57h+xSZFiUCyVfjzkqZ9l3gRne/b1TXzoi+MWWHRj1PMP6/mfHKjDft8UQed/f3m9kxwBNm9jN3XwfcAVzs7uvN7FOkpjcfa7L3KTIt6hqSvOQTzEqZXl0L7E4/D2re+ReA5WbWmH798ak2cPcXSU2+96X0ompgT7o76pOjivak1031PkWmRYlA8kWFme0a9fhzUl+eV5vZelKzS34wXfarpLpSHid1InfWpbuX/hT4lZk9AewFuqax6a3A281sGfC/gKeBB0kllhE/Ab6YHnK6gonfp8i0aPZRkYCYWZW796bnv78F2Oru3wo7LpGx1CIQCc5n0iePN5HqjvpeyPGIjEstAhGRAqcWgYhIgVMiEBEpcEoEIiIFTolARKTAKRGIiBS4/w9Hl3keA3WS2AAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.lr_find()\n",
    "learner.recorder.plot(skip_end=10,suggestion=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "lr = 3E-05"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.726409</td>\n",
       "      <td>1.706788</td>\n",
       "      <td>0.256109</td>\n",
       "      <td>0.743891</td>\n",
       "      <td>00:20</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAeoElEQVR4nO3de3Cdd33n8ff33CUdybJl2ZZlO1YcJ44THCdRTCAkGGjBDoRAyTIOUGgGxkvblMvOtkmn00K3zA5sZ3cYpgSPYT1pZ3MpDaFJGUMWdhPcJQQiE6PIdpw4tmPJcqybddeRzuW7f5xjI8u6HDnHVvTk85rR6DzP73ee8/05ykc/PVdzd0REZP4LzXUBIiJSGgp0EZGAUKCLiASEAl1EJCAU6CIiARGZqw9evHixr1x1GSf7UgyNZhjL5s5pX1geY0FZhGQiik2xjXQ2x0AqgwOJSIjRTI6h0fyyO4xlcpTHw+TccYd4JEQyHqE8FsEs38em2vg4xfYTEbnY9u7d2+XutZO1zVmg19at4PT7/wtlBu9bU8PBkwPctHohd1y3nOa2Pnb9v6OM5pwr6xfwhfetZcXCMjoHRllUEaMyEeGBp1/ln5taz25vpPC93KCuKsFYNkc660RCRlksTCwS4rXuYUZyTjgeIZtzRtJZ1tRWcOvaWjoGUhzpHAIgEjYGUhkyWSedzdE1OEptZZz66jIWVcQpi4VpqCmnobaC7sEx7t60ior4nP1TishbiJm9NmXbXJ2HXr/2Wo9+7Bt8/Q/exrZNq85rb+8d4acHTrFzzxFO9I5MsgW47cpa/uD6eq6tr6K5rY+Vi8p5W/0CEtHwpP1P9adobutj94sn6Roc5fpVC/n3Vzo59PoAtZVxrqhNEg4Z2ZwTCRuRcIhkLEJVWYRj3cMMjWboGRrjVH+K08Pps9uNho3l1WVEwyGWVsVZWpng/dcs44olFZTFItRUxKasaaLj3cP8+lgPW65dRjLAvyRyOScUurA/e7I5ZzCVoaosgulPJ3mLMbO97t44adtcBfqiy9b5rX/+PZ64913T9hvL5Hhq/+ukszmWViU41Z+ivXeELdfWccWS5CWq9nwnekfoGhglk8vx1P5THOkcxB2Odg/RPThG38i5gX/r2lqS8QjPH+shZMby6gQ3rV5EWTRMdXmUvpE0+9v7efpQB6l0jkQ0xK1ra6lKRPnlq1184u2ruKZ+AamxLADXrawmEjIqE1E6B0Y5dGqAo12DLKlMAPDjlpP86mgP6UyOdM5ZUV1GfyrD8uoEiUiY61dVk3Nn2YIyFlVEaVicpKGmggXl0WnHncs5+9p6GRrN0DU4yoH2fhaURXnHmsWsXZqkKpF/f8/QGP/+SicA77i85uwv5WcOdbL7xZO82jlIRSzCTQ2LMOCKpUkyWScRDbG6poLLa5M0t/Wydkkl71hTQ+/wGAdPDvDQr17jF4e76E9lSERDxCNhrltZjbtzzfIF9KfS1FeXnf3FvL6uiqGxDC+dHODgyX5ODaS4fuVC3n/NUlpO9BOLhLhjQx1LqhKl/hERuSjelIEer1vrf/6dx/naR942J59/MaWzOX59tIcjXUOEDI52DvHUgdcBWLukkoFUmr6RNC+fGjznfbWVcd5zVS23XLGYF4738i9NrQwVAvxCvOPyGq6uqyKdzXGyb4SF5THa+0YYHM2y/0QfOXdy4/7zhwyuXFpJdXmUa5cv4ObLa7jlisX0DI/x8K9eo28kzb7WXlpO9J99TywcOuf4R21lnIpYmPa+FGOZc4+LQP5YxE2rF3Hl0iRtp0c43j1MOpejtWeESMjIFo53jFceCzNc+HeoqYjxe1cvpaG2gq6BUQZSGV480Ucqk+VI5xDxwrGUiUIGi5Nx1tQmef5YD5lxA4+FQ3zu1gY+887VLFWwy5vcmzLQL7vqbf7LX/2a5dVlc/L5bwZHu4aoiIcZTeeoLo9SEYucsxvite4hWntGuOWKGk71j9J6ephkPMLwWIbfvNaL4/QMpVlSGef6VdU0LK7gRO8IqXSW9XULKItNvZsnlc4SDYfoHR7j9PAYhzuGOHiynxdae2nrGaatd4SxTI5YJEQ252QLAbg4GePP3ruWBWVRVi+uYN2ySlLpLD984QSvdAySSmfJ5pyKeIR3XbGY6rIoe17p4qplSQzjnVfUnP0rYmI98UiI4bEsp/pT/OZ4L5fVlNPRP8pzR7qpX1jG0qo4771q6ZR/RZzZxtBYllQ6y1gmx8m+FLFwiKuWVRKL5E/q6uhP8czLnVy2qJx4NMx39xxhd8tJYuEQ//G2y/noDSvoHBjlhlXVHO7M/9WzqCI25b+lu5PJOdFwiKHRDI+/cILnj/bgwI2rqtmwspr+kTTLq8s40N5P99AYACNjGQCurqtiNJPjnWtqqC6P4e680NpLeSzMumVV0/8QlVD34CgLyqIc7xlmYXn+WFU6m9/9ODyW5cW2PqrLozS39XG8Z5i1S/K7KBeUR1lUHuO6ldUXvcZ0Ok1bWxupVOqif9ZcSyQSrFixgmj03J/3N2WgNzY2elNT05x8tsxsNJPl+aOn+fnL+V1AH7m+nkUVMVbXlAdyv/Vr3UN89cn9PH2o87y2RDTEwvIYqxaVM5DKYAZXLElSmYjw4ol+jnQOMpDKsKwqQSbndA2OUl9dhrvT3ld88NQtSPCONTUc7hikua0PgOULEqxdWsllNeV8+Lrl3HjZwvP+/YdGMxw6NcDTL3XQdnqE265czE2rF7FiYfk5/dyd4bEsLSf6OHRqgFzO6RgYJZXO8eyrXbz0+sA5/cui4fwvcmPSv7YmioaNZDxCMhEhHgmzqDxGJpefFCTjEUYzOd6xpoZb1ixmXV0l8UiYM/lT7M/U0aNHqayspKamJpA/h2e4O693dNJ1uo81lzdwejhNTWFSURaLKNBFinGid4T/uvsgHf0prq6r4mjXEEsqE2RyOY73DBMyI2zGwZP9ZN3ZuLKaNbVJFifj/OJwF47zn99/FZsaFmFmtPYM85vjp6mtjNPaM8zKheVcXpukvW+EJZVxyqJhjnUPMZDK8PUfv8RAKkN1eZSPN65kLJMP2pdPDXJ6eIzhsSzrllXy7itrqSqLMpDK8IvDXbx4Ih/+IYMFZdGzB+wr4xGuXl5FJGS8fGqQbC53zsF8yO8Ci4VDbFxZTePqhWRzsGpROSf7RugeGqMsGsaA1tPDvHfdEhLRMGtqk1yxJMnhjkES0TBdg6O8fGqA9t4Ug6Np+kcyDKTS9AynScbDdA+OMZbJkck5x3uGgXz4J6Jh0tkcyXiUO66rI2zGSDrLse4houEQ1y5fwNsvX0TIjJ+/3MmB9n4+vzHB6suvIJmIUlWWP/aUSmcZGs1iQCIaJhHN7waMR0JUl8WmPfiezTl9I2Nn359MREjGIzj5Xae5nDMwmj/jbTSToywaZmlVPH+MprB7cCyTwx3CISPnTjhkDI5myOaceCTEYCpDOucY+T4V8QixSIiRsQyRcIiwGX0jafpTaUJmhAq/qFLpDKdaj7D9ydfPqfm1b3xIgS5SSl44/hC+wDN1Zmt4LMMT+9r55+db2dfaC+RD8apllXxg/TLWLk2yqaGG6rIozSf6aDrWw2vdwzSf6GMsk+Pa5VU4sKY2Sd2CBO9cU0Mm59QtSODOBZ9xNFvtvSPsa+2lua2PvpH8rqcXjvdypHOIaNiIRkKsWlTO8FiWwx2/O8YUDRtrapP8xduTLF21huy43DIzYmEjHAqRSmfJjWs7E45nwjQezV+XgkMOJzWWxYFoOETOf7drcTyD/LUu0TCj6Xx/w3CKy86wGbFIiHTOyWSn/ktnQVmUkNnZ62aS8QjHXn2Z53vLiYaN08NjlMci3PvetQp0kaAYzWQZHs2STESIhoN7sXffcJpfH+uhPBZm7dIkSyoTHDx4kHXr1nF6OD8zr4iHqUpEz+5+yeWc0UyWeCTM8Fj+NOPRbI6KWOTshYihQsAaUBGPkIyHz15HMjyWZaAwU45FQpgZFbEw4ZBhZoyMZc7OvsOhEGb5wA6FjNFMlmgoRNadZDz/3yaVzpKI5t/vhaBOZfLHmWLhEA6MZXNEQ0ZZ7PzTlA8ePMjVV199zrrp9qEH90RnkYCKR8LEI8Vd1zCfLSiP8vvrl5633symPEgdGheMyUSUZGL603AnqohHpr1IcHR4kH95+GH+5E/+ZJLW8z/rzLZuv/12Hn74YaqrqymfENzFXqNSjOD+ehcRKbHe3l4eeOCB89Zns9OfXrx7926qqy/+WUCaoYuIFOn+++/n1VdfZePGjUSjUZLJJHV1dezbt48DBw7wkY98hNbWVlKpFF/84hfZvn07AKtXr6apqYnBwUG2bt3Ku971Lp599lnq6+t54oknKCsrzenbCnQRmZf+9t/2c6C9f+aOs7B+eRVfueOaKdu//vWv09LSwr59+3jmmWf44Ac/SEtLCw0NDQDs2rWLRYsWMTIywk033cTHPvYxampqztnGK6+8wiOPPMJ3v/tdPv7xj/ODH/yAT33qUyWpX4EuInKBNm3adDbMAb71rW/xwx/+EIDW1lZeeeWV8wK9oaGBjRs3AnDjjTdy7NixktWjQBeReWm6mfSlUlFRcfb1M888w89+9jN++ctfUl5ezubNmye9ojUej599HQ6HGRmZ/OaDF0IHRUVEilRZWcnAwMCkbX19fSxcuJDy8nJeeuklnnvuuUtcnWboIiJFq6mp4ZZbbuHaa6+lrKyMpUt/d1rlli1b2LFjBxs2bOCqq67i5ptvvuT16cIiEZk3JrvQJshme2GRdrmIiATEjIFuZrvMrMPMWqbps9nM9pnZfjP7eWlLFBGRYhQzQ38Q2DJVo5lVAw8AH3b3a4D/UJrSRERkNmYMdHffA/RM0+UTwOPufrzQv6NEtYmIyCyUYh/6lcBCM3vGzPaa2aen6mhm282sycyaOjvPf5CAiIhcuFIEegS4Efgg8AHgr83sysk6uvtOd29098ba2toSfLSIiJxRikBvA37i7kPu3gXsAa4rwXZFROa1ZDIJQHt7O3fdddekfTZv3kypTuEuRaA/AdxqZhEzKwfeDhwswXZFRAJh+fLlPPbYYxf9c2a8UtTMHgE2A4vNrA34CoU7ubv7Dnc/aGY/AZqBHPA9d5/yFEcRkfnqvvvu47LLLjv7gIuvfvWrmBl79uzh9OnTpNNpvva1r3HnnXee875jx47xoQ99iJaWFkZGRrjnnns4cOAAV199dUnv5TJjoLv73UX0+Xvg70tSkYhIMX58P7z+Ymm3uextsPXrUzZv27aNL33pS2cD/fvf/z4/+clP+PKXv0xVVRVdXV3cfPPNfPjDHz77WLyJvvOd71BeXk5zczPNzc3ccMMNJStf93IRESnS9ddfT0dHB+3t7XR2drJw4ULq6ur48pe/zJ49ewiFQpw4cYJTp06xbNmySbexZ88evvCFLwCwYcMGNmzYULL6FOgiMj9NM5O+mO666y4ee+wxXn/9dbZt28ZDDz1EZ2cne/fuJRqNsnr16klvmzveVLP3N0r3chERmYVt27bx6KOP8thjj3HXXXfR19fHkiVLiEajPP3007z22mvTvv+2227joYceAqClpYXm5uaS1aYZuojILFxzzTUMDAxQX19PXV0dn/zkJ7njjjtobGxk48aNrFu3btr3//Ef/zH33HMPGzZsYOPGjWzatKlkten2uSIyb+j2ubp9rojIW4ICXUQkIBToIjKvzNVu4kvtQsapQBeReSORSNDd3R34UHd3uru7SSQSs3qfznIRkXljxYoVtLW18Va4/XYikWDFihWzeo8CXUTmjWg0SkNDw1yX8aalXS4iIgGhQBcRCQgFuohIQCjQRUQCYsZAN7NdZtZhZtM+tMLMbjKzrJlN/pwlERG5qIqZoT8IbJmug5mFgW8AT5WgJhERuQAzBrq77wF6Zuj2Z8APgI5SFCUiIrP3hvehm1k98FFgRxF9t5tZk5k1vRUuDBARuZRKcVD0m8B97p6dqaO773T3RndvrK2tLcFHi4jIGaW4UrQReLTwSKXFwO1mlnH3fy3BtkVEpEhvONDd/ex1uGb2IPAjhbmIyKU3Y6Cb2SPAZmCxmbUBXwGiAO4+435zERG5NGYMdHe/u9iNufsfvaFqRETkgulKURGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEDMGupntMrMOM2uZov2TZtZc+HrWzK4rfZkiIjKTYmboDwJbpmk/Crzb3TcAfwfsLEFdIiIyS8U8gm6Pma2epv3ZcYvPASveeFkiIjJbpd6H/lngx1M1mtl2M2sys6bOzs4Sf7SIyFtbyQLdzN5DPtDvm6qPu+9090Z3b6ytrS3VR4uICEXscimGmW0AvgdsdffuUmxTRERm5w3P0M1sFfA48Ifu/vIbL0lERC7EjDN0M3sE2AwsNrM24CtAFMDddwB/A9QAD5gZQMbdGy9WwSIiMrliznK5e4b2zwGfK1lFIiJyQXSlqIhIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiATFjoJvZLjPrMLOWKdrNzL5lZofNrNnMbih9mSIiMpNiZugPAlumad8KrC18bQe+88bLEhGR2Zox0N19D9AzTZc7gX/yvOeAajOrK1WBIiJSnFLsQ68HWscttxXWncfMtptZk5k1dXZ2luCjRUTkjFIEuk2yzifr6O473b3R3Rtra2tL8NEiInJGKQK9DVg5bnkF0F6C7YqIyCyUItCfBD5dONvlZqDP3U+WYLsiIjILkZk6mNkjwGZgsZm1AV8BogDuvgPYDdwOHAaGgXsuVrEiIjK1GQPd3e+eod2BPy1ZRSIickF0paiISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgFRVKCb2RYzO2Rmh83s/knaF5jZv5nZb81sv5npIRciIpfYjIFuZmHg28BWYD1wt5mtn9DtT4ED7n4d+acb/Xczi5W4VhERmUYxM/RNwGF3P+LuY8CjwJ0T+jhQaWYGJIEeIFPSSkVEZFrFBHo90Dpuua2wbrx/AK4G2oEXgS+6e27ihsxsu5k1mVlTZ2fnBZYsIiKTKSbQbZJ1PmH5A8A+YDmwEfgHM6s6703uO9290d0ba2trZ12siIhMrZhAbwNWjlteQX4mPt49wOOedxg4CqwrTYkiIlKMYgL9eWCtmTUUDnRuA56c0Oc48D4AM1sKXAUcKWWhIiIyvchMHdw9Y2b3Ak8BYWCXu+83s88X2ncAfwc8aGYvkt9Fc5+7d13EukVEZIIZAx3A3XcDuyes2zHudTvw/tKWJiIis6ErRUVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQBQV6Ga2xcwOmdlhM7t/ij6bzWyfme03s5+XtkwREZnJjE8sMrMw8G3g98k/MPp5M3vS3Q+M61MNPABscffjZrbkYhUsIiKTK2aGvgk47O5H3H0MeBS4c0KfTwCPu/txAHfvKG2ZIiIyk2ICvR5oHbfcVlg33pXAQjN7xsz2mtmnJ9uQmW03syYza+rs7LywikVEZFLFBLpNss4nLEeAG4EPAh8A/trMrjzvTe473b3R3Rtra2tnXayIiExtxn3o5GfkK8ctrwDaJ+nT5e5DwJCZ7QGuA14uSZUiIjKjYmbozwNrzazBzGLANuDJCX2eAG41s4iZlQNvBw6WtlQREZnOjDN0d8+Y2b3AU0AY2OXu+83s84X2He5+0Mx+AjQDOeB77t5yMQsXEZFzmfvE3eGXRmNjozc1Nc3JZ4uIzFdmttfdGydr05WiIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gERFGBbmZbzOyQmR02s/un6XeTmWXN7K7SlSgiIsWYMdDNLAx8G9gKrAfuNrP1U/T7BvknG4mIyCVWzAx9E3DY3Y+4+xjwKHDnJP3+DPgB0FHC+kREpEjFBHo90Dpuua2w7iwzqwc+CuyYbkNmtt3MmsysqbOzc7a1iojINIoJdJtk3cQHkX4TuM/ds9NtyN13unujuzfW1tYWW6OIiBQhUkSfNmDluOUVQPuEPo3Ao2YGsBi43cwy7v6vJalSRERmVEygPw+sNbMG4ASwDfjE+A7u3nDmtZk9CPxIYS4icmnNGOjunjGze8mfvRIGdrn7fjP7fKF92v3mIiJyaRQzQ8fddwO7J6ybNMjd/Y/eeFkiIjJbulJURCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAKCrQzWyLmR0ys8Nmdv8k7Z80s+bC17Nmdl3pSxURkenMGOhmFga+DWwF1gN3m9n6Cd2OAu929w3A3wE7S12oiIhMr5gZ+ibgsLsfcfcx4FHgzvEd3P1Zdz9dWHyO/IOkRUTkEiom0OuB1nHLbYV1U/ks8OPJGsxsu5k1mVlTZ2dn8VWKiMiMigl0m2SdT9rR7D3kA/2+ydrdfae7N7p7Y21tbfFViojIjIp5SHQbsHLc8gqgfWInM9sAfA/Y6u7dpSlPRESKVcwM/XlgrZk1mFkM2AY8Ob6Dma0CHgf+0N1fLn2ZIiIykxln6O6eMbN7gaeAMLDL3feb2ecL7TuAvwFqgAfMDCDj7o0Xr2wREZnI3CfdHX7RNTY2elNT05x8tojIfGVme6eaMOtKURGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEEUFupltMbNDZnbYzO6fpN3M7FuF9mYzu6H0pYqIyHRmDHQzCwPfBrYC64G7zWz9hG5bgbWFr+3Ad0pcp4iIzKCYGfom4LC7H3H3MeBR4M4Jfe4E/snzngOqzayuxLWKiMg0ZnxINFAPtI5bbgPeXkSfeuDk+E5mtp38DB5g1MxaZlXt/LQY6JrrIi6Rt8pYNc5gmW/jvGyqhmIC3SZZN/HJ0sX0wd13AjsBzKxpqgedBslbZZzw1hmrxhksQRpnMbtc2oCV45ZXAO0X0EdERC6iYgL9eWCtmTWYWQzYBjw5oc+TwKcLZ7vcDPS5+8mJGxIRkYtnxl0u7p4xs3uBp4AwsMvd95vZ5wvtO4DdwO3AYWAYuKeIz955wVXPL2+VccJbZ6waZ7AEZpzmft6ubhERmYd0paiISEAo0EVEAmJOAn2mWwnMJ2a2y8w6xp9Tb2aLzOynZvZK4fvCcW1/WRj3ITP7wNxUPXtmttLMnjazg2a238y+WFgfqLGaWcLMfm1mvy2M828L6wM1zjPMLGxmL5jZjwrLQR3nMTN70cz2mVlTYV3wxurul/SL/IHVV4HLgRjwW2D9pa6jhOO5DbgBaBm37r8B9xde3w98o/B6fWG8caCh8O8QnusxFDnOOuCGwutK4OXCeAI1VvLXVCQLr6PAr4CbgzbOceP9T8DDwI8Ky0Ed5zFg8YR1gRvrXMzQi7mVwLzh7nuAngmr7wT+sfD6H4GPjFv/qLuPuvtR8mcFbbokhb5B7n7S3X9TeD0AHCR/NXCgxup5g4XFaOHLCdg4AcxsBfBB4HvjVgdunNMI3FjnItCnuk1AkCz1wnn4he9LCusDMXYzWw1cT372GrixFnZD7AM6gJ+6eyDHCXwT+AsgN25dEMcJ+V/K/9vM9hZuQQIBHGsxl/6XWlG3CQioeT92M0sCPwC+5O79ZpMNKd91knXzYqzungU2mlk18EMzu3aa7vNynGb2IaDD3fea2eZi3jLJujf9OMe5xd3bzWwJ8FMze2mavvN2rHMxQ38r3Cbg1Jm7TRa+dxTWz+uxm1mUfJg/5O6PF1YHcqwA7t4LPANsIXjjvAX4sJkdI7/b871m9r8I3jgBcPf2wvcO4Ifkd6EEbqxzEejF3EpgvnsS+Ezh9WeAJ8at32ZmcTNrIH//+F/PQX2zZvmp+P8EDrr7/xjXFKixmlltYWaOmZUBvwe8RMDG6e5/6e4r3H01+f8H/6+7f4qAjRPAzCrMrPLMa+D9QAsBHOtcHXG+nfxZEq8CfzXXR4bf4FgeIX+b4DT53+yfBWqA/wO8Uvi+aFz/vyqM+xCwda7rn8U430X+z85mYF/h6/agjRXYALxQGGcL8DeF9YEa54Qxb+Z3Z7kEbpzkz6j7beFr/5nMCeJYdem/iEhA6EpREZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRALi/wM2SOjegfsZHgAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.fit_one_cycle(1, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.save('second_cycle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed_all(seed)\n",
    "learner.load('second_cycle');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.freeze_to(-3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.694062</td>\n",
       "      <td>1.690368</td>\n",
       "      <td>0.254154</td>\n",
       "      <td>0.745846</td>\n",
       "      <td>00:22</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZRc5Xnn8e9TW++bepFa3QK1kAQSIAQ0ApvFskNiCTAyCbGFsR1rHGuIQ2wzfwxkMo4zg89MbJ/kZBwvOoqHQ5gwyB6MDcNgFG9CjhE2LRaphdCC0NJqSb2p97W6nvmjSnKr1Uu1KKnpy+9zTp+ue+9bdZ+3l1+99da9t8zdERGRmS803QWIiEhmKNBFRAJCgS4iEhAKdBGRgFCgi4gERGS6dpxbWOLDeWUsriggK6rnFRGRdGzfvr3F3cvH2jZtgR4tnk3FJ/6Ob/27FXxg8Zi1iYjIKGZ2aLxt0z40PtLWO90liIgEwrQFeiKRPKFJgS4ikhnTFuhDwwkA3jjWOV0liIgEyrTNoTsQC4f41b4WdjZ0cGV10XSVIiIzxNDQEA0NDfT39093KedddnY21dXVRKPRtO8zbYEOcP+HFvLYtkP812d38X/ue/90ljIpd8cdQiGbtG177yCdfXE27zqOGWRFQlQW5TC3OIdFs/NJuJMVCV+AquFkzyD98WFe2NNM90Cc8oIs2nuHuHlRGZ39cX75ZhNH2nqZW5xDY0cfebEI5QVZRMJGe+8QpXkxFlbkU5IXo6lzgGjYKM6NsXxeMeE0fhYimdTQ0EBBQQHz58/HLLh/f+5Oa2srDQ0N1NTUpH2/aQ30OUXZ/Mn7LubvfrqXtp5BZuXFprMcAAbjCfae6KJ7IE7IjKKcKJtePszm+uNkR8N87Lp5fOSquURCxm/ebuP7Lx+mZ2CYS8rzqTvURtiMAy09E+4jOxpiWVUyEGflxzje0c8l5XlEwiEWlOWx8tIK5hRlk5819q+nf2iY5q4BBuIJasry+Lf9Lfyg7gidfUPcfW0111xUwq7GTv77T3ZzqHXi9yhCBqX5WTR3DVCcmxwJtPcOARANG0PDY1+8rSw/xhVVRVw3fxafu3kBsUjmZ+/cneGEEwmHcHfiCceAt5p7eP1IO0vnFrK0spDO/iHaegZpbO+neyDOtrda2Hm0g0g4RFFOlA8sLuf2KyspGefvq39oOPXEGz693+buAZo6BzjS1ktb7yDHO/oZjCcYiCc43NZLwp33LShl7XUXUZQbpat/iF/vbyUnFuamhWV6sjtP+vv7Ax/mAGZGaWkpzc3NU7vfdF1tMatykT+1eStzi7O56zsv8o/3XM1Hrpqb1n13NnRQmBPh4tK80+sG4sM0dQ7wizebiIZDzMqL8tqRDsryY/zTrw5QO38WlYXZVBbncO/1F5EdTf7zHuvoY+veZl5v6OBQaw+vHW6nZ3D4rH3mxsL0jrF+TmE2g8MJ2noGKcqJsqA8j/dfUkosHObWpRVUl+QyMDTM0fY+9p3o5sjJXtp6BlNPGsO09w6SlxVhf1P3GY87vzSXL9+xlAXl+bR2D/DYtkPML81l674WXm9oZ/SvrSA7cjpwTqkqzqGiMIv8rAjrbpzP7MJsmrsGqC7J5df7W8jPivB7Syoozo3RNzhMViREKGQMxIfp7ItTnBvlZM8gB1p66BtKbndPBv5zO4+x+3gnB5p7WFSRTzzhNHcN0Dc0zLySHNYsr2JJZQFLK4sozotysKWHnoFhBuLDOHBDTSk5sfFfpfQNDrP+f9Xx6uF2qktyaDjZR/dA/Kx2+VkRegbjZ/w8zGB2QTbVJTm0dA9wsLWXguwIX/jQIn61v4WDLT3csriMu66u5rdvt/GtX+yjP56guiQHdzhysvesny8kX2llRUJkRcMMDA3T2R8nLxZmVn6MI219p9vVXlzCR6+u4tYls5ldmDVp+Bzv6KfuUBtLKwtp6xnkid8eoao4m+sXlPJGYydVJTksrSykODdKV3+cjr4hqopzxn2CCrLdu3ezZMmS6S7jghmrv2a23d1rx2o/rYG+ecuvuWlhGdf/t5/zvktK+dYnrhm3/ZY9TcQiITr7hrjvX14hOxrii7+3mFcOn+REZz97T3TRP5QY875ZkRAluTE6+4foHRzmrquruP9DC3nwyR3UHToJJAOxLD+Lqy8q5pZF5VQUZNE3NExrzyA3LSxjbnEOAIdbe/lJ/THCIaOyKIcbF5YSChnDw37O/2DuTs/gMHmxMM/XH+ff9rewZU8zR9v7xmz//ktKue3KSmKREDsa2rl8bhF/eE0VhrH3RBevHj5JNBzij66tJho+v+97P19/jIef3U1RTpQbFpSSHQ3x6uF2th1onfB+BdkRyvOzKM6NUlmUQ/WsHO675RKKc6N8ffMenq8/ztstySeLqpIc5pfmUZoXw0k+ud6yuJz6ox28erid0vwYlUXZVBRmMxRPcOPCMvJSr27cnV2NnXxx06u81dxDeUEWV1UX8at9Laef/G5dUsFlcwo51NbLcCL5qqcsP4vKomyqS3LJjYWpKcs7K5h3NnTw+G8O0d47RFlBjFsWldPeO8TfPv8mbT2DAFSXJKfacqJhyguyeOD3F1OWH2PzrhP87I0T1Dd2cKB54ld0p4RDRiI19RcOGZfNKaD24hJyYhF++sZxOvvjLCzP583jnZTlZ/HwR6/ghgWlnPof33uimz0nurhsTgELyvIImZ01hdg9ECc7EiJynv9uztVMDnR3Z9idSOh3P9tE6ncTMmMwPoyZYXD65z+jAv3X235D7fxZ/Ocf7+SH24+y/cu3khs7c5rhQHM3Oxo6+NL3XwOSf8iz8mJkR0NnjIp+f+lsblhQysmeQa6sLqJ/aJi5xTnEwiGWVRed/mf85s/38fc/3QtAcW6Uj9fO47YrK7miquhd9TJ5MJ7gZ7tP8Nu32zjS1stX77qC5q4BFs8uOP3q4t2spXuAxvY+dh7toOFkH+X5WdSU5ZEVCdHcPcDmXcfpGxymvW+I9t4h3h41TVV7cQl/dG0196y4KCP1JBLO0fY+5hRlEw2HaOke4JVDJynJi3Hd/FkZ2ccp7s7eE938667j7DjaQWffEH1Dw+w+1nl62ih11C65sTC3LpnNbVfOobVnkNkF2VxRVcSbxzsZTjjL5xWz90Q3R9v72H2sk0jIWDq3kD3Hu3j1cDvbD50k4c77F5ZRlh/j5YNtVBfncryzn4OtPVQWZtPSPUhxbpSmroGzaq0syqaiIIuewWHCZuxv7j79pBEyuHlROXOLs7mkPJ/Dbb00dQ7Q2T/E0spC7rhqLksrC3mruZuKgiwKsqPnZeptpEwHes9AnJM9g4RCRiwSYmAogXtycGYG7tDZP0QsHEpu7+ni+5ue4NOfXc/JniGyoiHKC7LAk68M23uH6BmMU5AVIRYJ0TeUoKmzn3//ybv5+re/R05+IYXZUeIJZzCeIJ5IYGZEw8bgiFfX2dEwebEwxw+/xZ7+It53SSkVBVnsa+rm6otK3p2B/tor21lSWchvDrTy8Y0v8T/WLmfN8ioAhhPOhhfe4hub9wDJP7zra2YRi4T44q2LKc/P4lBrDxeV5nKkrY9Lys8eQY3F3Xnm9UaOtvex6vI5LCjPP6/9lPRs3dvMi2+18ti2g6y8tJx/vOead9UTbCYcbOnhx68d5cW3WllUkc+6G2tYWPHO/v46Uu93FOWeeSREZ/8Q/7T1AIdak/P9ZkZVcQ4fvLScukMn+cdf7CMnGubmReU0dfWTFQkTDYeYW5xNLBwiLytCa88A33/5CPHE714V1JTlkZcVYfexzjMCCCAWCbHq8jmsumIO4VDy/afra2ad83y3e/JJ+LFth6gpy6O7P87ywh4uWXQpbT2DZEdDlOVnTXmAMxAfprs/Tmd/nK7+IcIhwz05Wg6bkUjteyxHjxzmLz7zcZ76+TbCIWM44UTDIQaG4kTC4dMj7pGyo2HCISNkRjhktPcOkh0JE4uEGPbk/RMJPz0FmXCnb3CY3sFhjh16i889c+yMxzv0tTvenYG+r/51LirNJZFwbv76L0m48/cfW04kbPzxhm2n215fM4uHP3oFi2cXTEutcuEMxIcv2BFAMrmO3iFikRB7TnQxNzWtBdDVP8T3Xz7C0681cu/1F3G8s59fvNnEgeaeM97reN+CUv7wmiriCefai0vY1djB4tkFFGYnn4Dmzco93bZ/aJite5vZuq+Z3ce6aOsZPOuV2z/dWcnsixYAYBiOkxuLUJIbJScWJpFw2nqHcHfysiJEQ5Y8wCFkREIhTvYO0j+UfC8sZEZ+VoR5s3IImTEQTxCLJMO1eyDO4HDi9IERHX1DREPGZz51L88/9ywLFy0mOytGdm4es8oq2F2/k59v287nPvVxmo430tfXx32fv5/P/unnyM2KsKCmhrq6Orq7u1m9ejU33XQTL774IlVVVTz99NPk5OSc9bNPuFO/6w2GCip56UAbLd0DXF9TyuorK9+dgd64r57S/CwA3mjs5HOP1Z0xb1ySG+XRdSu4al7xtNQoIlMTH07wyuF2sqMhfvt2G9/Z8tbp9xPGMr80l4TDoop89jV1c7itl5DBtan3BioKsrj9ykrK8rOoLsnh6MH9zFuwiLxYmIf/3xvsbOggPuxnjYzNbMxR9qnpjVMj5tGWzi3kKx+5fNx6Dx48yB133EF9fT1btmzh9ttvp76+/vShhW1tbcyaNYu+vj6uu+46XnjhBUpLS5k/f/7pQF+4cCF1dXUsX76cj33sY9x555188pOfHHN/U51Dn9bDFvNGHJa3dG4h//rALfyg7gg/393EH15TxW1XVs6I+WIRSYqEQ6yoSb4nsay6mLuurmLPiS5yYxF+9EoDKy+tYOfRDoYTyUNSD7R0MzTs/GpfM/1DCf7TbZdx19XVyXnpMRxPTeVAcoQdDYeIhpOj2XjCCZGcGjIznOTUScjsjDcfM2nFihVnHCf+zW9+kx/96EcAHDlyhH379lFaWnrGfWpqali+fDkA1157LQcPHsxYPdMW6Eby6JOR8rIirLuxhnU3pn8gvYi8e5XmZ/H+1Kvw5alX2h+8rOKsdsc6+ujsi3PpnPSnVScaSV8oeXm/O3R6y5Yt/OxnP2Pbtm3k5uaycuXKMc9ozcr63ZNVOBymr2/so9nOxfRdPjccCvzJASKSnsqiHCpnwNU/CgoK6OrqGnNbR0cHJSUl5Obm8uabb/LSSy9d4OqmMdCn8kwsIvJuUFpayo033sgVV1xBTk4Os2fPPr1t1apVbNiwgWXLlnHppZdyww03XPD6pu1N0draWq+rq5uWfYvIzDSTTyw6F1N9U/TdeTqYiIhM2aSBbmaPmFmTmdVP0Galmb1mZrvM7IXMligiIulIZ4T+KLBqvI1mVgx8B7jT3S8H/jgzpYmIyFRMGujuvhVom6DJJ4Cn3P1wqn1ThmoTEZEpyMQc+mKgxMy2mNl2M/v0eA3NbL2Z1ZlZ3VSv8ysiIhPLRKBHgGuB24EPA182s8VjNXT3je5e6+615eXlGdi1iIickolAbwCed/ced28BtgJXZeBxRURmtPz85NU0Gxsbufvuu8dss3LlSjJ1CHcmAv1p4GYzi5hZLnA9sDsDjysiEghz587lySefPO/7mfRMUTN7AlgJlJlZA/AVIArg7hvcfbeZPQ/sABLA99x93EMcRURmqgcffJCLL76Yz3/+8wD8zd/8DWbG1q1bOXnyJENDQ3z1q19lzZo1Z9xv5FUa+/r6WLduHW+88QZLliy5sNdycfd70mjzDeAbGalIRCQdP3kIju/M7GPOuRJW/+24m9euXcuXvvSl04H+gx/8gOeff54HHniAwsJCWlpauOGGG7jzzjvHvVbVd7/7XXJzc9mxYwc7duzgmmvG/+jNqZrWy+eKiMwkV199NU1NTTQ2NtLc3ExJSQmVlZU88MADbN26lVAoxNGjRzlx4gRz5swZ8zG2bt3KF77wBQCWLVvGsmXLMlafAl1EZqYJRtLn0913382TTz7J8ePHWbt2LY8//jjNzc1s376daDTK/Pnzx7xs7kjn60qzupaLiMgUrF27lk2bNvHkk09y991309HRQUVFBdFolF/+8pccOnRowvvfcsstPP744wDU19ezY8eOjNWmEbqIyBRcfvnldHV1UVVVRWVlJffeey8f+chHqK2tZfny5Vx22WUT3v/P/uzPWLduHcuWLWP58uWsWLEiY7Xp8rkiMmPo8rm6fK6IyHuCAl1EJCAU6CIyo0zXNPGFdi79VKCLyIyRnZ1Na2tr4EPd3WltbSU7O3tK99NRLiIyY1RXV9PQ0MB74fLb2dnZVFdXT+k+CnQRmTGi0Sg1NTXTXca7lqZcREQCQoEuIhIQCnQRkYBQoIuIBMSkgW5mj5hZk5lN+KEVZnadmQ2b2difsyQiIudVOiP0R4FVEzUwszDwNWBzBmoSEZFzMGmgu/tWoG2SZn8B/BBoykRRIiIyde94Dt3MqoC7gA1ptF1vZnVmVvdeODFARORCysSbov8APOjuw5M1dPeN7l7r7rXl5eUZ2LWIiJySiTNFa4FNqY9UKgNuM7O4u/84A48tIiJpeseB7u6nz8M1s0eBZxXmIiIX3qSBbmZPACuBMjNrAL4CRAHcfdJ5cxERuTAmDXR3vyfdB3P3z7yjakRE5JzpTFERkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhCTBrqZPWJmTWZWP872e81sR+rrRTO7KvNliojIZNIZoT8KrJpg+9vAB9x9GfAwsDEDdYmIyBSl8xF0W81s/gTbXxyx+BJQ/c7LEhGRqcr0HPpngZ+Mt9HM1ptZnZnVNTc3Z3jXIiLvbRkLdDP7IMlAf3C8Nu6+0d1r3b22vLw8U7sWERHSmHJJh5ktA74HrHb31kw8poiITM07HqGb2UXAU8Cn3H3vOy9JRETOxaQjdDN7AlgJlJlZA/AVIArg7huAvwZKge+YGUDc3WvPV8EiIjK2dI5yuWeS7X8K/GnGKhIRkXOiM0VFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQmISQPdzB4xsyYzqx9nu5nZN81sv5ntMLNrMl+miIhMJp0R+qPAqgm2rwYWpb7WA99952WJiMhUTRro7r4VaJugyRrgMU96CSg2s8pMFSgiIunJxBx6FXBkxHJDat1ZzGy9mdWZWV1zc3MGdi0iIqdkItBtjHU+VkN33+jute5eW15enoFdi4jIKZkI9AZg3ojlaqAxA48rIiJTkIlAfwb4dOpolxuADnc/loHHFRGRKYhM1sDMngBWAmVm1gB8BYgCuPsG4DngNmA/0AusO1/FiojI+CYNdHe/Z5LtDvx5xioSEZFzojNFRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJiLQC3cxWmdkeM9tvZg+Nsb3IzP6vmb1uZrvMTB9yISJygU0a6GYWBr4NrAaWAveY2dJRzf4ceMPdryL56UZ/Z2axDNcqIiITSGeEvgLY7+4H3H0Q2ASsGdXGgQIzMyAfaAPiGa1UREQmlE6gVwFHRiw3pNaN9C1gCdAI7AS+6O6J0Q9kZuvNrM7M6pqbm8+xZBERGUs6gW5jrPNRyx8GXgPmAsuBb5lZ4Vl3ct/o7rXuXlteXj7lYkVEZHzpBHoDMG/EcjXJkfhI64CnPGk/8DZwWWZKFBGRdKQT6C8Di8ysJvVG51rgmVFtDgO/B2Bms4FLgQOZLFRERCYWmayBu8fN7H5gMxAGHnH3XWZ2X2r7BuBh4FEz20lyiuZBd285j3WLiMgokwY6gLs/Bzw3at2GEbcbgT/IbGkiIjIVOlNURCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiAREWoFuZqvMbI+Z7Tezh8Zps9LMXjOzXWb2QmbLFBGRyUz6iUVmFga+Dfw+yQ+MftnMnnH3N0a0KQa+A6xy98NmVnG+ChYRkbGlM0JfAex39wPuPghsAtaMavMJ4Cl3Pwzg7k2ZLVNERCaTTqBXAUdGLDek1o20GCgxsy1mtt3MPj3WA5nZejOrM7O65ubmc6tYRETGlE6g2xjrfNRyBLgWuB34MPBlM1t81p3cN7p7rbvXlpeXT7lYEREZ36Rz6CRH5PNGLFcDjWO0aXH3HqDHzLYCVwF7M1KliIhMKp0R+svAIjOrMbMYsBZ4ZlSbp4GbzSxiZrnA9cDuzJYqIiITmXSE7u5xM7sf2AyEgUfcfZeZ3ZfavsHdd5vZ88AOIAF8z93rz2fhIiJyJnMfPR1+YdTW1npdXd207FtEZKYys+3uXjvWNp0pKiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQKQV6Ga2ysz2mNl+M3tognbXmdmwmd2duRJFRCQdkwa6mYWBbwOrgaXAPWa2dJx2XyP5yUYiInKBpTNCXwHsd/cD7j4IbALWjNHuL4AfAk0ZrE9ERNKUTqBXAUdGLDek1p1mZlXAXcCGiR7IzNabWZ2Z1TU3N0+1VhERmUA6gW5jrBv9QaT/ADzo7sMTPZC7b3T3WnevLS8vT7dGERFJQySNNg3AvBHL1UDjqDa1wCYzAygDbjOzuLv/OCNViojIpNIJ9JeBRWZWAxwF1gKfGNnA3WtO3TazR4FnFeYiIhfWpIHu7nEzu5/k0Sth4BF332Vm96W2TzhvLiIiF0Y6I3Tc/TnguVHrxgxyd//MOy9LRESmSmeKiogEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEGkFupmtMrM9ZrbfzB4aY/u9ZrYj9fWimV2V+VJFRGQikwa6mYWBbwOrgaXAPWa2dFSzt4EPuPsy4GFgY6YLFRGRiaUzQl8B7Hf3A+4+CGwC1oxs4O4vuvvJ1OJLJD9IWkRELqB0Ar0KODJiuSG1bjyfBX4y1gYzW29mdWZW19zcnH6VIiIyqXQC3cZY52M2NPsgyUB/cKzt7r7R3Wvdvba8vDz9KkVEZFLpfEh0AzBvxHI10Di6kZktA74HrHb31syUJyIi6UpnhP4ysMjMaswsBqwFnhnZwMwuAp4CPuXuezNfpoiITGbSEbq7x83sfmAzEAYecfddZnZfavsG4K+BUuA7ZgYQd/fa81e2iIiMZu5jToefd7W1tV5XVzct+xYRmanMbPt4A2adKSoiEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAiKtQDezVWa2x8z2m9lDY2w3M/tmavsOM7sm86WKiMhEJg10MwsD3wZWA0uBe8xs6ahmq4FFqa/1wHczXKeIiEwinRH6CmC/ux9w90FgE7BmVJs1wGOe9BJQbGaVGa5VREQmMOmHRANVwJERyw3A9Wm0qQKOjWxkZutJjuABBsysfkrVzkxlQMt0F3GBvFf6qn4Gy0zr58XjbUgn0G2MdaM/WTqdNrj7RmAjgJnVjfdBp0HyXuknvHf6qn4GS5D6mc6USwMwb8RyNdB4Dm1EROQ8SifQXwYWmVmNmcWAtcAzo9o8A3w6dbTLDUCHux8b/UAiInL+TDrl4u5xM7sf2AyEgUfcfZeZ3ZfavgF4DrgN2A/0AuvS2PfGc656Znmv9BPeO31VP4MlMP0097OmukVEZAbSmaIiIgGhQBcRCYhpCfTJLiUwk5jZI2bWNPKYejObZWY/NbN9qe8lI7b9Zarfe8zsw9NT9dSZ2Twz+6WZ7TazXWb2xdT6QPXVzLLN7Ldm9nqqn/8ltT5Q/TzFzMJm9qqZPZtaDmo/D5rZTjN7zczqUuuC11d3v6BfJN9YfQtYAMSA14GlF7qODPbnFuAaoH7Euq8DD6VuPwR8LXV7aaq/WUBN6ucQnu4+pNnPSuCa1O0CYG+qP4HqK8lzKvJTt6PAb4AbgtbPEf39D8D/Bp5NLQe1nweBslHrAtfX6Rihp3MpgRnD3bcCbaNWrwH+OXX7n4GPjli/yd0H3P1tkkcFrbgghb5D7n7M3V9J3e4CdpM8GzhQffWk7tRiNPXlBKyfAGZWDdwOfG/E6sD1cwKB6+t0BPp4lwkIktmeOg4/9b0itT4QfTez+cDVJEevgetrahriNaAJ+Km7B7KfwD8A/xFIjFgXxH5C8kn5X81se+oSJBDAvqZz6n+mpXWZgICa8X03s3zgh8CX3L3TbKwuJZuOsW5G9NXdh4HlZlYM/MjMrpig+Yzsp5ndATS5+3YzW5nOXcZY967v5wg3unujmVUAPzWzNydoO2P7Oh0j9PfCZQJOnLraZOp7U2r9jO67mUVJhvnj7v5UanUg+wrg7u3AFmAVwevnjcCdZnaQ5LTnh8zsXwhePwFw98bU9ybgRySnUALX1+kI9HQuJTDTPQP8Ser2nwBPj1i/1syyzKyG5PXjfzsN9U2ZJYfi/xPY7e5/P2JToPpqZuWpkTlmlgPcCrxJwPrp7n/p7tXuPp/k/+Av3P2TBKyfAGaWZ2YFp24DfwDUE8C+Ttc7zreRPEriLeCvpvud4XfYlydIXiZ4iOQz+2eBUuDnwL7U91kj2v9Vqt97gNXTXf8U+nkTyZedO4DXUl+3Ba2vwDLg1VQ/64G/Tq0PVD9H9XklvzvKJXD9JHlE3eupr12nMieIfdWp/yIiAaEzRUVEAshCIrgAAAAnSURBVEKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJiP8PPa8+kXaZnx0AAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.fit_one_cycle(1, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.save('third_cycle')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "seed_all(seed)\n",
    "learner.load('third_cycle');"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "learner.unfreeze()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: left;\">\n",
       "      <th>epoch</th>\n",
       "      <th>train_loss</th>\n",
       "      <th>valid_loss</th>\n",
       "      <th>accuracy</th>\n",
       "      <th>error_rate</th>\n",
       "      <th>time</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>0</td>\n",
       "      <td>1.676122</td>\n",
       "      <td>1.672161</td>\n",
       "      <td>0.279570</td>\n",
       "      <td>0.720430</td>\n",
       "      <td>00:56</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAD4CAYAAAD8Zh1EAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjEsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+j8jraAAAgAElEQVR4nO3deZScdZ3v8fe3qvct3ektSXcn6UB2SEJoQhSBII4kiGR04BhwzeBwQXFhzlxhjl7lqvc6KHq8cxUj42XQM0BGkQEmIosKhAECdCBkIXvI0tl6SbrT+1L1vX9UJXbaXqpDJZ1++LzO6dNVz/Orp76/Xj71q189i7k7IiIy+oVGugAREUkOBbqISEAo0EVEAkKBLiISEAp0EZGASBmpJ87IzXfPKWb6uFzSwnpdERFJxNq1a+vdvbi/dSMW6J5TzPjP/ph/uXkhF08pHKkyRERGFTPbM9C6ER8aHzrWMdIliIgEwsgHepMCXUQkGUY80A8q0EVEkmLE5tCPO9DYPtIliMgo0d3dTU1NDR0dwR8IZmRkUF5eTmpqasKPGdFAz01PYevh5pEsQURGkZqaGnJzc5k8eTJmNtLlnDbuTkNDAzU1NVRWVib8uBGdcrl4SiF7Gtpoau8eyTJEZJTo6OigsLAw0GEOYGYUFhYO+53IiAb6wiljAXh646GRLENERpGgh/lxp9LPEQ30D88ax/yJ+Xz/6S1EozqNr4jIuzGigZ6fncoNCyZS39J1Vsylu/tJLyydPRE6uiM8t6WWHz6zlYdf28vR1q4RrFBERlJjYyP33nvvsB939dVX09jYeBoqOtmIfiianZbCJecWAbB6Wx0zx+cN2HZ9TSM9UWf+xAIaWjrJSkshMy2c0PN09URJSxn4tetIaxf/47GNvPpOA62dEdJSQozLy+BwcweNbSfP73/r8U18+n2TKC/IpK65k6KcdJYtqCArbcR3GJIkc3ca4i/gmalhmjt6iLqzYX8TO+tamDg2iw/OKCErLQV35429jTS0dPLBGSWk6HQWgXQ80G+99Va6IlHCZqSEQ0QiEcLhMJFolJ6IEwoZqeEQ3ZEoLR09/Pujj+MOR1o7GZOZSntXhM6eKK1dEY5PrHT2RAEIGWSnpzAmM/G9W44bsRQKmREOGRPyM5k9IY+nNh3iv11+Tr9tu3qiXPuTlwD41MKJPPzaPqYUZfPQ3y2kODcdgPqWTjbUNPHUxkOEQsa1cycQDhkb9jdx9++38L8+dh4fnj2OnXUtXFCRf2J+6sXtddz0y2q6eqLMnpBHyIwZ43I5dKyDc0qyyUlPobIoh0mFWaSFQzy96RD/+tI79J4h+uEzW7l23gS2HGpm9oQ8yvKzKMhKpXRMBrMn5PHSjnqmleYyvTT3rPtHd3de2tHA7oZW0sIhCrLTmFqSw8SxWZiNjvnKtq4edtS2ALCnoY0N+5vYd6SNd+pbyc1I4dtLz2PGuFwAHlu3nzf3NrJ0XhkXTipg7Z4j/Pvr+yjITmNeeT5H2rrYfriF/Y3trN1zlCNDvCObXprLrAl5vLWvkV31rQBcXDmW5ZdUMntCHhPyMwmHkvszbOnsIRL1E//wDS2drK9p4o29R8lOTyEtHKKyOJtF04pP+v09vm4/r+xsYEJ+JhdXjiUnI4Xi3HRKcjPo6I6w6cAxnttSS3pKiGnjcqksyuac4hx6olFSQyFCSe7H2cDd6YpEiUadkBmtXREi0Sh5Gamkp8YGjD2RWNC2d0f42h13sHPnTmadP5dwOIWs7GwqyiawYcN6XnvjLa7/m4+zf38NnZ2dfPqmW/j4jZ8FYMn75vDQ756jrbWVL37mei64aCHr1r5GSel4fnL/Q2RmZZISDtHdE8WJ/Y5rj3VwpLmTu37+Ch+dO4EpxdnsPzr4bt42Upegm3beXN+28S0AfvKn7dzzzDZe//qHTgQ0QGtnD5//ZTXt3RHW7fvz25XM1DARd9LCIUry0snNSGV3fWvCe8v896umc8vl5/C1R9bz2zdqAPhEVQV3XzcnocfvbWgj4k5hThpbDzXzxQffoLa5c8jH5aSn8MEZJTgwc3wuXT1RZo7PY3ppLhGP/YOOzUo7Y/84dc2dfOOxDTy96fBfrAuHjKy0MDnpKRxt62JcXgZZaSl0dEfIzUzlvAl5/N2lU5hclH3S4zq6I7R1RcjNSCG114uXu9PY1k1mWpim9m5K8zKGrG9PQytr9xwlPyuVtHCY/KxUaps7eHzdATYfPMb0cXlMGJPBqvUH2d/neIa8jBQqxmZxsKmDI61dFOWkUZafyVs1TSfajMvL4NCxDnLSU+jsidAd+fP/QmleOpdPK2ZaaeyFoLUzwticNNLCRsiMtJQQ9S1d/ORP2zna1k1uRgq3XH4OeZmp/NOTm2ntigBQlp/J3IoxXDV7HLPG5zE1vr3BtHdFSAnHRnjtXbF3jCGDo23dfGXlm6zZ1YCZUV6QiQHv1LfS30dQl08r5vyyMby4vY4DTR3UNXeSnhI6MRI8bkpxNgca2+nojv7lRuJKctNJDYdISwlRXpDJFdNL+JsLy09pFDkcdc2d/OqV3RTnptPWFeHignamTJ1OXXMnP39hJ7sbWgkNc9ARcccdolGnJxqldwRWFmfzd5dOif2Ow7EXsbaunhPr9+/by5c+9wmeeO5Vtq9bww3Xf4xHnn2Z8omTAGg6epTSkiK8u5Nr/+oynvj9HygfX8Ksaefywkuv0NLawrzZM/nji69wcdV8PnXjMv566VI+9alPAbGazKAn6tS3dLJj61b+/tl66nrly567r1nr7lX99W3ERuh5GX/+Q7hsWjH3PLONl3fWs3ReGQCPvlHD3//6rRNtrppdyg+un8vv1h/k0qlFNLR0sfL1fVTvPsJb+xpZMHkstyyaQnlBFvmZqfxxSy1v7j3KvIoCrj5/HL/feIi65k7W7WvkB09v5ZlNh3irpokvXnEO8ycWDOsEYRMLs07cvmjyWJ647QPsPdJGUU4aE/Iz6Y5EaWrvZt+Rdl59p4HKomxaOyP8bsMBXt99hO5IlP9860C/255bPob//fHzmVKUk/CUUiJ6IlF6ok7UnXfqW9l04BjfiwfPPy6ZwccuKKMrEqW+pYstB4/xTkMrLR09NLZ309YZ+94TcYpy0om485u1NTz46l5K89K56QOVvLGnkYg7r+5q4FhH7B/g3JIczinO5vCxTtbXNBJ1MAN3uHBSAdddWM6s8XnMnpB30juXPQ2tvLk39nvqG9QQ28bkwmz+8PZh2rsjzBiXyw+vn0tuRgotnT18ZM540sIhzIy65k7+bc0eXtxex56GNr710VlcX1XBg2v2sL22hcmFWSy/pBIz2F3fRlqKUVmUQyjBdyfL3z+ZLYeamTEu98QL8fUXlvP2wWNs2t/EM28f5tVdR3hyQ2xPru8snc0nLppIWkqIQ00ddEei/GlLLQ+9upfuaOwt/MGmDgqyU8lNT2Xb4WZ6ok5mapj27li4L7toIo5zpLWLrp4oH5xRwodmljK7bAxHWrroikT405Za7nlmGy9sq+Oc+Gj9vLIx3HjxRLYfbuGh1/ZQnJNBVyTC1kMtLJpWwsVTxjJ/YgHhkLH54DH2H23nR89uoyg3jZ5ILARzMlLYUdvCi9vr+f7TW7hsajE761oozEln1vg8Lpo89sS7uwn5mYzNThvW36m7s3H/MVZvr+O5LbVU7zl60vp/uXY84XiId/ZEYz+TcIiUkJ34fUXjCX086B2w+PKO7shJAW4GaSmxvxV3Z0xGKtNKcznY1EFnTwSikJ+VRk8kSk5GCt2N6aSlhJg+LpfarDQuXrCAS+fPIjUce6H8/s9/xKonHgfg0IH9HD20l+mTJ2AGORmp0JNKZWUll7/vIgAuqqpi9+7dJ+o5/jeUGjbGj8mkMTed/7rjCnbUtlDf0sXEsVlMuXvgn9+IjdCrqqq8uroagEjUufC7z1JZlE006ieNohZOGcv3Pj6HSWOz+h25Hmnt4vF1+7lhwUQyUocOwC2HjrHk/7yIO3zlyqnc/lfTktepBLk71XuOUpqbweu7j9DeHRvR7jvSxg+f3YY7zKvI5//ecAEVY7OG3mA/jnV087XfrKcrEsWANbsa6I44aSkhWjpjgTu3Ip8fXj+Hc0uGHjX2VXusg9+sreHpTYdYH/99leVnMn9SAWX5mbywrY7OngghMzJTw8yryKcwJ41INPbO6j/e3H9iimJyYRZfvnIq00pziUSdG/5lDW1dEcZmp/EPH55OxdhMeqJOZ3eUwpw0ygsyGT8mE4iNaM72qYC2rh7e2NPIP/9xO6/tPtJvm9K82NTH8ZH5oaYOxuVlMKU4G3fITAvz5r5G/vaSyScGPUM5fKyDqPuJn9Wp6IlECfcKy+M27m/iwVf38vzWWioKsthZ13Li84bjwiHj61fP5MaLJ7LlUDOleen8prqGcMjIi7+D++sLyk78324/3My3V73Ni9vrAZgxLpe55fl8aFYp6SkhphRnU7dvF+MnnUthTuyF4kBj+4l35sffOXV0x94dhePTut2RKMQDOzUcoiw/k9SUEKnxfg1nSmz37t1cc801bNy4keeff5577rmHVatWAfD888/zjW98g2eeeYasrCwWLVrEXXfdxaJFi5g8eTLV1dW0tLSceDzAPffcQ0tLC3fddVe/z7d582Zmzpx50jIzO/tG6L2FQ8bfXlLJj57dRl5GrKQ55WP41d8uYExm6qAjpbHZaSy/JPEjqWaMy+OFf7iC0jHppKckbwQ8HGbGRZNj++D3Hu0DfGBqMb96eTerNhxk0T3Pc+HEAj7z/knUNXdy3YXl5Mbf2bjHRmgv7Wzg5y/spKm9m4smj+Xzl1Yye8IYvvfkZp7a9Of9+68+fxxjMtM42NTOFdNLmD0hj/kTC045DEvyMvjiFedy6+Xn8F876hk/JuOk6YQ7l8wY9PG3ffBcNh04xo7aFr77u80nvRsLGfzmlvcxc3weOemD/4me7WEOkJWWwgemFnFRZQGr3jrId373No1t3cyryOfKGSVMLc3h8mklSX1HBiQ0rTWUgT7zOa9sDN/7+PknLTvU1MH+xnZqj3VgZjz02l6+veptvr3q7QG3/43HNpKRGiYcMprau8lNT+HOJTP4+PwySnL/sv7mQ2HGjfnz8kmF2bR3RWjp7KG9K0JrVw856SnkZqTS1ROhpTPC2Jx0QgYpoRAFWanv6nOs3Nxcmpv73yOvqamJgoICsrKy2LJlC2vWrDnl5zlVZ0WgA3xh0TnMrcjnfVMKOXysg9K8jEH3THk3+obo2WReRT7zPjGPL185lbuf2sJr7xzhtofeBOD+l97hE1UVvLSjgdaunpNGxrMm5PGHzYd5bN1+Kguz2VXfyvJLJvPNa2YRdZL+wdxxoZBx2bR+z7U/KDPjvLIxnFc2ho/MGc/K1/dxtLWLZ98+zOcvrTzxghck6Slh/ubCcpbOm4ATG0GOhhekRI0bk3FS2F41u5QVL+xi7Z4jLJpewis7G7jp0krqmzsZPyaTo21dPLe1FoDfrT9ITnoK/7r8IqqG+bvPTAsn/cVwIIWFhVxyySWcd955ZGZmUlpaemLd4sWLWbFiBXPmzGH69OksXLjwjNTU21kx5SID23ekjcfe3E/EnR//YfuJ5cW56XyiqoJ5FflcNq2YtJQQTe3d/Oz5nax4YScA1d/4EEU56QNtWuSs0dTeTXckOuTfa39TEEE2KqdcZGAVY7P40pVTAXj/OUXUNndw4aSCfudFx2SmcueSGXzu/ZNpau9WmMuocbr3lnmvGDLQzex+4Bqg1t3PG6DNIuDHQCpQ7+6XJ7NIiVlQmdhb0b5vfUXkvSGRSeoHgMUDrTSzfOBe4Fp3nw1cn5zSRERkOIYMdHdfDfS/r1XMjcCj7r433r42SbWJiMgwJGM3kmlAgZk9b2ZrzewzAzU0s5vNrNrMquvq6pLw1CIiclwyAj0FuBD4CHAV8D/MrN+jddz9Pnevcveq4uLh7+omIiIDS0ag1wBPuXuru9cDq4G5SdiuiMiolpOTA8CBAwe47rrr+m2zaNEikrULdzIC/XHgUjNLMbMs4GJgcxK2KyISCBMmTOCRRx457c+TyG6LDwOLgCIzqwG+RWz3RNx9hbtvNrOngPVAFPiFu288fSWLiIyMO+64g0mTJvGFL3wBgLvuugszY/Xq1Rw9epTu7m6++93vsnTp0pMe1/scMO3t7Sxfvpy3336bmTNn0t4++Clxh2PIQHf3GxJo8wPgB0mpSEQkEb+/Ew5tSO42x50PS/5pwNXLli3jq1/96olA//Wvf81TTz3F7bffTl5eHvX19SxcuJBrr712wHNQ/exnPyMrK4v169ezfv165s+fn7TydaSoiEiCLrjgAmprazlw4AB1dXUUFBQwfvx4br/9dlavXk0oFGL//v0cPnyYcePG9buN1atX8+UvfxmAOXPmMGdOYtdhSIQCXURGp0FG0qfTddddxyOPPMKhQ4dYtmwZDz74IHV1daxdu5bU1FQmT55MR0fHoNs4XVcCO7uuhyYicpZbtmwZK1eu5JFHHuG6666jqamJkpISUlNTee6559izZ8+gj7/ssst48MEHAdi4cSPr169PWm0aoYuIDMPs2bNpbm6mrKyM8ePH88lPfpKPfvSjVFVVMW/ePGbMGPxaALfeeivLly9nzpw5zJs3jwULFiStNp0+V0RGDZ0+d/DT52rKRUQkIBToIiIBoUAXkVFlpKaJz7RT6acCXURGjYyMDBoaGgIf6u5OQ0MDGRnDu1CN9nIRkVGjvLycmpoa3gun387IyKC8vHxYj1Ggi8iokZqaSmVl5UiXcdbSlIuISEAo0EVEAkKBLiISEAp0EZGAGDLQzex+M6s1s0EvWmFmF5lZxMz6v86SiIicVomM0B8AFg/WwMzCwN3A00moSURETsGQge7uq4EjQzT7EvBboDYZRYmIyPC96zl0MysDPgasSKDtzWZWbWbV74UDA0REzqRkfCj6Y+AOd48M1dDd73P3KnevKi4uTsJTi4jIcck4UrQKWBm/pFIRcLWZ9bj7Y0nYtoiIJOhdB7q7nzgO18weAFYpzEVEzrwhA93MHgYWAUVmVgN8C0gFcPch581FROTMGDLQ3f2GRDfm7p97V9WIiMgp05GiIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gExJCBbmb3m1mtmW0cYP0nzWx9/OtlM5ub/DJFRGQoiYzQHwAWD7L+HeByd58DfAe4Lwl1iYjIMCVyxaLVZjZ5kPUv97q7Bih/92WJiMhwJXsO/Sbg9wOtNLObzazazKrr6uqS/NQiIu9tSQt0M7uCWKDfMVAbd7/P3avcvaq4uDhZTy0iIiQw5ZIIM5sD/AJY4u4NydimiIgMz7seoZvZROBR4NPuvu3dlyQiIqdiyBG6mT0MLAKKzKwG+BaQCuDuK4BvAoXAvWYG0OPuVaerYBER6V8ie7ncMMT6zwOfT1pFIiJySnSkqIhIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJiCED3czuN7NaM9s4wHozs382sx1mtt7M5ie/TBERGUoiI/QHgMWDrF8CTI1/3Qz87N2XJSIiwzVkoLv7auDIIE2WAr/ymDVAvpmNT1aBIiKSmGTMoZcB+3rdr4kv+wtmdrOZVZtZdV1dXRKeWkREjktGoFs/y7y/hu5+n7tXuXtVcXFxEp5aRESOS0ag1wAVve6XAweSsF0RERmGZAT6E8Bn4nu7LASa3P1gErYrIiLDkDJUAzN7GFgEFJlZDfAtIBXA3VcATwJXAzuANmD56SpWREQGNmSgu/sNQ6x34ItJq0hERE6JjhQVEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgEgp0M1tsZlvNbIeZ3dnP+jFm9p9m9paZbTIzXeRCROQMGzLQzSwM/BRYAswCbjCzWX2afRF4293nEru60Q/NLC3JtYqIyCASGaEvAHa4+y537wJWAkv7tHEg18wMyAGOAD1JrVRERAaVSKCXAft63a+JL+vtJ8BM4ACwAfiKu0f7bsjMbjazajOrrqurO8WSRUSkP4kEuvWzzPvcvwpYB0wA5gE/MbO8v3iQ+33uXuXuVcXFxcMuVkREBpZIoNcAFb3ulxMbife2HHjUY3YA7wAzklOiiIgkIpFAfx2YamaV8Q86lwFP9GmzF7gSwMxKgenArmQWKiIig0sZqoG795jZbcDTQBi43903mdkt8fUrgO8AD5jZBmJTNHe4e/1prFtERPoYMtAB3P1J4Mk+y1b0un0A+HBySxMRkeHQkaIiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiAREQoFuZovNbKuZ7TCzOwdos8jM1pnZJjN7IblliojIUIa8wIWZhYGfAn9F7Pqir5vZE+7+dq82+cC9wGJ332tmJaerYBER6V8iI/QFwA533+XuXcBKYGmfNjcSu0j0XgB3r01umSIiMpREAr0M2Nfrfk18WW/TgAIze97M1prZZ5JVoIiIJCaRa4paP8u8n+1cCFwJZAKvmNkad9920obMbgZuBpg4ceLwqxURkQElMkKvASp63S8HDvTT5il3b3X3emA1MLfvhtz9Pnevcveq4uLiU61ZRET6kUigvw5MNbNKM0sDlgFP9GnzOHCpmaWYWRZwMbA5uaWKiMhghpxycfceM7sNeBoIA/e7+yYzuyW+foW7bzazp4D1QBT4hbtvPJ2Fi4jIycy973T4mVFVVeXV1dUj8twiIqOVma1196r+1ulIURGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCjQRUQCQoEuIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAkFupktNrOtZrbDzO4cpN1FZhYxs+uSV6KIiCRiyEA3szDwU2AJMAu4wcxmDdDubmKXqhMRkTMskRH6AmCHu+9y9y5gJbC0n3ZfAn4L1CaxPhERSVAigV4G7Ot1vya+7AQzKwM+BqwYbENmdrOZVZtZdV1d3XBrFRGRQSQS6NbPsr5Xlv4xcIe7RwbbkLvf5+5V7l5VXFycaI0iIpKAlATa1AAVve6XAwf6tKkCVpoZQBFwtZn1uPtjSalSRESGlEigvw5MNbNKYD+wDLixdwN3rzx+28weAFYpzEVEzqwhA93de8zsNmJ7r4SB+919k5ndEl8/6Ly5iIicGYmM0HH3J4En+yzrN8jd/XPvviwRERkuHSkqIhIQCnQRkYBQoIuIBIQCXUQkIBToIiIBoUAXEQkIBbqISEAo0EVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAJBToZrbYzLaa2Q4zu7Of9Z80s/Xxr5fNbG7ySxURkcEMGehmFgZ+CiwBZgE3mNmsPs3eAS539znAd4D7kl2oiIgMLpER+gJgh7vvcvcuYCWwtHcDd3/Z3Y/G764hdiFpERE5gxIJ9DJgX6/7NfFlA7kJ+H1/K8zsZjOrNrPqurq6xKsUEZEhJRLo1s8y77eh2RXEAv2O/ta7+33uXuXuVcXFxYlXKSIiQ0rkItE1QEWv++XAgb6NzGwO8Atgibs3JKc8ERFJVCIj9NeBqWZWaWZpwDLgid4NzGwi8CjwaXfflvwyRURkKEOO0N29x8xuA54GwsD97r7JzG6Jr18BfBMoBO41M4Aed686fWWLiEhf5t7vdPhpV1VV5dXV1SPy3CIio5WZrR1owKwjRUVEAkKBLiISEAp0EZGAUKCLiASEAl1EJCAU6CIiAaFAFxEJCAW6iEhAKNBFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCQgFuohIQCQU6Ga22My2mtkOM7uzn/VmZv8cX7/ezOYnv1QRERnMkIFuZmHgp8ASYBZwg5nN6tNsCTA1/nUz8LMk1ykiIkNIZIS+ANjh7rvcvQtYCSzt02Yp8CuPWQPkm9n4JNcqIiKDGPIi0UAZsK/X/Rrg4gTalAEHezcys5uJjeABOs1s47CqHZ2KgPqRLuIMea/0Vf0MltHWz0kDrUgk0K2fZX2vLJ1IG9z9PuA+ADOrHuhCp0HyXuknvHf6qn4GS5D6mciUSw1Q0et+OXDgFNqIiMhplEigvw5MNbNKM0sDlgFP9GnzBPCZ+N4uC4Emdz/Yd0MiInL6DDnl4u49ZnYb8DQQBu53901mdkt8/QrgSeBqYAfQBixP4LnvO+WqR5f3Sj/hvdNX9TNYAtNPc/+LqW4RERmFdKSoiEhAKNBFRAJiRAJ9qFMJjCZmdr+Z1fbep97MxprZs2a2Pf69oNe6f4z3e6uZXTUyVQ+fmVWY2XNmttnMNpnZV+LLA9VXM8sws9fM7K14P/9nfHmg+nmcmYXN7E0zWxW/H9R+7jazDWa2zsyq48uC11d3P6NfxD5Y3QlMAdKAt4BZZ7qOJPbnMmA+sLHXsu8Dd8Zv3wncHb89K97fdKAy/nMIj3QfEuzneGB+/HYusC3en0D1ldgxFTnx26nAq8DCoPWzV3//HngIWBW/H9R+7gaK+iwLXF9HYoSeyKkERg13Xw0c6bN4KfDL+O1fAn/da/lKd+9093eI7RW04IwU+i65+0F3fyN+uxnYTOxo4ED11WNa4ndT419OwPoJYGblwEeAX/RaHLh+DiJwfR2JQB/oNAFBUurx/fDj30viywPRdzObDFxAbPQauL7GpyHWAbXAs+4eyH4CPwa+BkR7LQtiPyH2ovyMma2Nn4IEAtjXRA79T7aEThMQUKO+72aWA/wW+Kq7HzPrr6Es/isAAAGESURBVEuxpv0sGxV9dfcIMM/M8oH/MLPzBmk+KvtpZtcAte6+1swWJfKQfpad9f3s5RJ3P2BmJcCzZrZlkLajtq8jMUJ/L5wm4PDxs03Gv9fGl4/qvptZKrEwf9DdH40vDmRfAdy9EXgeWEzw+nkJcK2Z7SY27flBM/s3gtdPANz9QPx7LfAfxKZQAtfXkQj0RE4lMNo9AXw2fvuzwOO9li8zs3QzqyR2/vjXRqC+YbPYUPz/AZvd/Ue9VgWqr2ZWHB+ZY2aZwIeALQSsn+7+j+5e7u6Tif0P/sndP0XA+glgZtlmlnv8NvBhYCMB7OtIfeJ8NbG9JHYCXx/pT4bfZV8eJnaa4G5ir+w3AYXAH4Ht8e9je7X/erzfW4ElI13/MPr5AWJvO9cD6+JfVwetr8Ac4M14PzcC34wvD1Q/+/R5EX/eyyVw/SS2R91b8a9NxzMniH3Vof8iIgGhI0VFRAJCgS4iEhAKdBGRgFCgi4gEhAJdRCQgFOgiIgGhQBcRCYj/D+rge4aHrIvKAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "learner.fit_one_cycle(1, max_lr=slice(lr*0.95**num_groups, lr), moms=(0.8, 0.9))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "def get_preds_as_nparray(ds_type) -> np.ndarray:\n",
    "    \"\"\"\n",
    "    the get_preds method does not yield the elements in order by default\n",
    "    we borrow the code from the RNNLearner to resort the elements into their correct order\n",
    "    \"\"\"\n",
    "    preds = learner.get_preds(ds_type)[0].detach().cpu().numpy()\n",
    "    sampler = [i for i in databunch.dl(ds_type).sampler]\n",
    "    reverse_sampler = np.argsort(sampler)\n",
    "    return preds[reverse_sampler, :]\n",
    "\n",
    "test_preds = get_preds_as_nparray(DatasetType.Test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.20171092"
      ]
     },
     "execution_count": 51,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_preds[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub = pd.read_csv('../input/mh-20-fakenews/Participants_Data_WH20/sample submission.csv')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 53,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     0    1    2    3    4    5\n",
       "0  0.0  0.0  0.0  0.0  0.0  0.0\n",
       "1  0.0  0.0  0.0  0.0  0.0  0.0\n",
       "2  0.0  0.0  0.0  0.0  0.0  0.0\n",
       "3  0.0  0.0  0.0  0.0  0.0  0.0\n",
       "4  0.0  0.0  0.0  0.0  0.0  0.0"
      ]
     },
     "execution_count": 53,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.head(5)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "for i in range(0,len(sub)):\n",
    "    for j in range(0,6):\n",
    "        sub.iloc[i,j] = test_preds[i][j]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>0</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>1267.000000</td>\n",
       "      <td>1267.000000</td>\n",
       "      <td>1267.000000</td>\n",
       "      <td>1267.000000</td>\n",
       "      <td>1267.000000</td>\n",
       "      <td>1267.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.148097</td>\n",
       "      <td>0.165343</td>\n",
       "      <td>0.235676</td>\n",
       "      <td>0.216950</td>\n",
       "      <td>0.055687</td>\n",
       "      <td>0.178249</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.059333</td>\n",
       "      <td>0.067998</td>\n",
       "      <td>0.055963</td>\n",
       "      <td>0.076056</td>\n",
       "      <td>0.054022</td>\n",
       "      <td>0.079338</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.033923</td>\n",
       "      <td>0.040243</td>\n",
       "      <td>0.071848</td>\n",
       "      <td>0.014927</td>\n",
       "      <td>0.002580</td>\n",
       "      <td>0.005969</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.103805</td>\n",
       "      <td>0.114366</td>\n",
       "      <td>0.198205</td>\n",
       "      <td>0.161599</td>\n",
       "      <td>0.019678</td>\n",
       "      <td>0.117852</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.144837</td>\n",
       "      <td>0.155475</td>\n",
       "      <td>0.226422</td>\n",
       "      <td>0.210871</td>\n",
       "      <td>0.039923</td>\n",
       "      <td>0.172606</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.184257</td>\n",
       "      <td>0.208244</td>\n",
       "      <td>0.262318</td>\n",
       "      <td>0.274702</td>\n",
       "      <td>0.075427</td>\n",
       "      <td>0.236196</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>0.435589</td>\n",
       "      <td>0.483786</td>\n",
       "      <td>0.474071</td>\n",
       "      <td>0.442819</td>\n",
       "      <td>0.491675</td>\n",
       "      <td>0.448781</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 0            1            2            3            4  \\\n",
       "count  1267.000000  1267.000000  1267.000000  1267.000000  1267.000000   \n",
       "mean      0.148097     0.165343     0.235676     0.216950     0.055687   \n",
       "std       0.059333     0.067998     0.055963     0.076056     0.054022   \n",
       "min       0.033923     0.040243     0.071848     0.014927     0.002580   \n",
       "25%       0.103805     0.114366     0.198205     0.161599     0.019678   \n",
       "50%       0.144837     0.155475     0.226422     0.210871     0.039923   \n",
       "75%       0.184257     0.208244     0.262318     0.274702     0.075427   \n",
       "max       0.435589     0.483786     0.474071     0.442819     0.491675   \n",
       "\n",
       "                 5  \n",
       "count  1267.000000  \n",
       "mean      0.178249  \n",
       "std       0.079338  \n",
       "min       0.005969  \n",
       "25%       0.117852  \n",
       "50%       0.172606  \n",
       "75%       0.236196  \n",
       "max       0.448781  "
      ]
     },
     "execution_count": 55,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub.to_csv('MH20_FastAI_v2d.csv', index = False)# add part 3 1epoch - lb: 1.6831326571397953 - best one"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
